add evaluations course and update models table

This commit is contained in:
Colt Steele MacBook
2024-09-04 16:45:46 -06:00
parent cf2979dc88
commit b81598db86
93 changed files with 15212 additions and 18 deletions

View File

@@ -0,0 +1,17 @@
import re
def get_assert(output, context):
topic = context["vars"]["topic"]
goal_count = int(context["vars"]["count"])
pattern = fr'(^|\s)\b{re.escape(topic)}\b'
actual_count = len(re.findall(pattern, output.lower()))
pass_result = goal_count == actual_count
result = {
"pass": pass_result,
"score": 1 if pass_result else 0,
"reason": f"Expected {topic} to appear {goal_count} times. Actual: {actual_count}",
}
return result