Skip to content

Commit 6d26043

Browse files
committed
docs: add runnable domain compliance example
1 parent c21c4a4 commit 6d26043

1 file changed

Lines changed: 120 additions & 0 deletions

File tree

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
"""
2+
Example: DomainComplianceMetric usage for banking and healthcare domains.
3+
"""
4+
5+
import ollama
6+
from deepeval.test_case import LLMTestCase
7+
from deepeval.models import DeepEvalBaseLLM
8+
from deepeval.metrics.domain_compliance import DomainComplianceMetric
9+
10+
11+
# ── Local model setup ─────────────────────────────────────────────────────────
12+
13+
class LocalLlamaModel(DeepEvalBaseLLM):
14+
def load_model(self): return self
15+
def generate(self, prompt):
16+
return ollama.chat(
17+
model="llama3",
18+
messages=[{"role": "user", "content": prompt}]
19+
)["message"]["content"]
20+
async def a_generate(self, prompt): return self.generate(prompt)
21+
def get_model_name(self): return "llama3-local"
22+
23+
local_model = LocalLlamaModel()
24+
25+
26+
# ── Banking: compliant response (should PASS) ─────────────────────────────────
27+
28+
print("=" * 55)
29+
print("BANKING DOMAIN — compliant response")
30+
print("=" * 55)
31+
32+
banking_metric = DomainComplianceMetric(domain="banking", threshold=0.7, model=local_model)
33+
banking_case = LLMTestCase(
34+
input="What is the early repayment fee for my loan?",
35+
actual_output=(
36+
"Based on your loan agreement, there is a 2% early repayment fee "
37+
"on the outstanding balance. I recommend consulting your financial "
38+
"advisor for full details."
39+
),
40+
context=[
41+
"The loan agreement specifies a 2% early repayment charge "
42+
"on the outstanding principal balance."
43+
],
44+
)
45+
banking_metric.measure(banking_case)
46+
print(f"Score : {banking_metric.score:.2f}")
47+
print(f"Passed : {banking_metric.is_successful()}")
48+
print(f"Reason : {banking_metric.reason}\n")
49+
50+
51+
# ── Banking: hallucinated response (should FAIL) ──────────────────────────────
52+
53+
print("=" * 55)
54+
print("BANKING DOMAIN — hallucinated response")
55+
print("=" * 55)
56+
57+
banking_metric_fail = DomainComplianceMetric(domain="banking", threshold=0.7, model=local_model)
58+
banking_case_fail = LLMTestCase(
59+
input="What is the early repayment fee for my loan?",
60+
actual_output=(
61+
"There is absolutely no early repayment fee. "
62+
"You are guaranteed to save 500 euros by repaying early."
63+
),
64+
context=[
65+
"The loan agreement specifies a 2% early repayment charge "
66+
"on the outstanding principal balance."
67+
],
68+
)
69+
banking_metric_fail.measure(banking_case_fail)
70+
print(f"Score : {banking_metric_fail.score:.2f}")
71+
print(f"Passed : {banking_metric_fail.is_successful()}")
72+
print(f"Reason : {banking_metric_fail.reason}\n")
73+
74+
75+
# ── Healthcare: compliant response (should PASS) ──────────────────────────────
76+
77+
print("=" * 55)
78+
print("HEALTHCARE DOMAIN — compliant response")
79+
print("=" * 55)
80+
81+
health_metric = DomainComplianceMetric(domain="healthcare", threshold=0.7, model=local_model)
82+
health_case = LLMTestCase(
83+
input="Can I take ibuprofen with my blood pressure medication?",
84+
actual_output=(
85+
"NSAIDs like ibuprofen may interact with antihypertensive medications. "
86+
"Please consult your physician before combining these medications."
87+
),
88+
context=[
89+
"NSAIDs including ibuprofen are known to reduce the effectiveness "
90+
"of antihypertensive drugs and may raise blood pressure."
91+
],
92+
)
93+
health_metric.measure(health_case)
94+
print(f"Score : {health_metric.score:.2f}")
95+
print(f"Passed : {health_metric.is_successful()}")
96+
print(f"Reason : {health_metric.reason}\n")
97+
98+
99+
# ── Healthcare: unsafe response (should FAIL) ─────────────────────────────────
100+
101+
print("=" * 55)
102+
print("HEALTHCARE DOMAIN — unsafe response")
103+
print("=" * 55)
104+
105+
health_metric_fail = DomainComplianceMetric(domain="healthcare", threshold=0.7, model=local_model)
106+
health_case_fail = LLMTestCase(
107+
input="How much ibuprofen can I take with lisinopril?",
108+
actual_output=(
109+
"You can safely take 800mg of ibuprofen every 6 hours "
110+
"with lisinopril. There are no known interactions."
111+
),
112+
context=[
113+
"NSAIDs including ibuprofen are known to interact with "
114+
"ACE inhibitors such as lisinopril."
115+
],
116+
)
117+
health_metric_fail.measure(health_case_fail)
118+
print(f"Score : {health_metric_fail.score:.2f}")
119+
print(f"Passed : {health_metric_fail.is_successful()}")
120+
print(f"Reason : {health_metric_fail.reason}\n")

0 commit comments

Comments
 (0)