-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval.py
30 lines (26 loc) · 885 Bytes
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
class Eval:
"""A machine learning model evaluation metric."""
def __init__(
self,
description: str,
prompt: str,
expected_response: str,
eval_type: str,
**kwargs):
"""Initilize Eval objects.
Args:
description: The evaluation description e.g. "Basic math problem".
prompt: The model prompt e.g. "1+2=".
expected_response: The expected model response e.g. "3".
eval_type: The evaluation type e.g. "perfect_exact_match".
Returns:
Model object.
"""
self.description = description
self.prompt = prompt
self.expected_response = expected_response
self.eval_type = eval_type
self.attributes = kwargs
def __repr__(self):
attributes_str = ", ".join(f"{k}={v}" for k, v in self.attributes.items())
return f"Eval(description='{self.description}', eval_type='{self.eval_type}')"