rename folder to prompt_evaluations and update README

This commit is contained in:
Colt Steele MacBook
2024-09-04 17:06:50 -06:00
parent 8174d9d587
commit c2dd47ec9b
94 changed files with 8 additions and 11 deletions

View File

@@ -0,0 +1,17 @@
import re
def get_assert(output, context):
topic = context["vars"]["topic"]
goal_count = int(context["vars"]["count"])
pattern = fr'(^|\s)\b{re.escape(topic)}\b'
actual_count = len(re.findall(pattern, output.lower()))
pass_result = goal_count == actual_count
result = {
"pass": pass_result,
"score": 1 if pass_result else 0,
"reason": f"Expected {topic} to appear {goal_count} times. Actual: {actual_count}",
}
return result