add evaluations course and update models table
This commit is contained in:
17
evaluations/07_prompt_foo_custom_graders/count.py
Normal file
17
evaluations/07_prompt_foo_custom_graders/count.py
Normal file
@@ -0,0 +1,17 @@
|
||||
import re
|
||||
|
||||
def get_assert(output, context):
|
||||
topic = context["vars"]["topic"]
|
||||
goal_count = int(context["vars"]["count"])
|
||||
pattern = fr'(^|\s)\b{re.escape(topic)}\b'
|
||||
|
||||
actual_count = len(re.findall(pattern, output.lower()))
|
||||
|
||||
pass_result = goal_count == actual_count
|
||||
|
||||
result = {
|
||||
"pass": pass_result,
|
||||
"score": 1 if pass_result else 0,
|
||||
"reason": f"Expected {topic} to appear {goal_count} times. Actual: {actual_count}",
|
||||
}
|
||||
return result
|
||||
Reference in New Issue
Block a user