1+ """
2+ Example: DomainComplianceMetric usage for banking and healthcare domains.
3+ """
4+
5+ import ollama
6+ from deepeval .test_case import LLMTestCase
7+ from deepeval .models import DeepEvalBaseLLM
8+ from deepeval .metrics .domain_compliance import DomainComplianceMetric
9+
10+
11+ # ── Local model setup ─────────────────────────────────────────────────────────
12+
13+ class LocalLlamaModel (DeepEvalBaseLLM ):
14+ def load_model (self ): return self
15+ def generate (self , prompt ):
16+ return ollama .chat (
17+ model = "llama3" ,
18+ messages = [{"role" : "user" , "content" : prompt }]
19+ )["message" ]["content" ]
20+ async def a_generate (self , prompt ): return self .generate (prompt )
21+ def get_model_name (self ): return "llama3-local"
22+
23+ local_model = LocalLlamaModel ()
24+
25+
26+ # ── Banking: compliant response (should PASS) ─────────────────────────────────
27+
28+ print ("=" * 55 )
29+ print ("BANKING DOMAIN — compliant response" )
30+ print ("=" * 55 )
31+
32+ banking_metric = DomainComplianceMetric (domain = "banking" , threshold = 0.7 , model = local_model )
33+ banking_case = LLMTestCase (
34+ input = "What is the early repayment fee for my loan?" ,
35+ actual_output = (
36+ "Based on your loan agreement, there is a 2% early repayment fee "
37+ "on the outstanding balance. I recommend consulting your financial "
38+ "advisor for full details."
39+ ),
40+ context = [
41+ "The loan agreement specifies a 2% early repayment charge "
42+ "on the outstanding principal balance."
43+ ],
44+ )
45+ banking_metric .measure (banking_case )
46+ print (f"Score : { banking_metric .score :.2f} " )
47+ print (f"Passed : { banking_metric .is_successful ()} " )
48+ print (f"Reason : { banking_metric .reason } \n " )
49+
50+
51+ # ── Banking: hallucinated response (should FAIL) ──────────────────────────────
52+
53+ print ("=" * 55 )
54+ print ("BANKING DOMAIN — hallucinated response" )
55+ print ("=" * 55 )
56+
57+ banking_metric_fail = DomainComplianceMetric (domain = "banking" , threshold = 0.7 , model = local_model )
58+ banking_case_fail = LLMTestCase (
59+ input = "What is the early repayment fee for my loan?" ,
60+ actual_output = (
61+ "There is absolutely no early repayment fee. "
62+ "You are guaranteed to save 500 euros by repaying early."
63+ ),
64+ context = [
65+ "The loan agreement specifies a 2% early repayment charge "
66+ "on the outstanding principal balance."
67+ ],
68+ )
69+ banking_metric_fail .measure (banking_case_fail )
70+ print (f"Score : { banking_metric_fail .score :.2f} " )
71+ print (f"Passed : { banking_metric_fail .is_successful ()} " )
72+ print (f"Reason : { banking_metric_fail .reason } \n " )
73+
74+
75+ # ── Healthcare: compliant response (should PASS) ──────────────────────────────
76+
77+ print ("=" * 55 )
78+ print ("HEALTHCARE DOMAIN — compliant response" )
79+ print ("=" * 55 )
80+
81+ health_metric = DomainComplianceMetric (domain = "healthcare" , threshold = 0.7 , model = local_model )
82+ health_case = LLMTestCase (
83+ input = "Can I take ibuprofen with my blood pressure medication?" ,
84+ actual_output = (
85+ "NSAIDs like ibuprofen may interact with antihypertensive medications. "
86+ "Please consult your physician before combining these medications."
87+ ),
88+ context = [
89+ "NSAIDs including ibuprofen are known to reduce the effectiveness "
90+ "of antihypertensive drugs and may raise blood pressure."
91+ ],
92+ )
93+ health_metric .measure (health_case )
94+ print (f"Score : { health_metric .score :.2f} " )
95+ print (f"Passed : { health_metric .is_successful ()} " )
96+ print (f"Reason : { health_metric .reason } \n " )
97+
98+
99+ # ── Healthcare: unsafe response (should FAIL) ─────────────────────────────────
100+
101+ print ("=" * 55 )
102+ print ("HEALTHCARE DOMAIN — unsafe response" )
103+ print ("=" * 55 )
104+
105+ health_metric_fail = DomainComplianceMetric (domain = "healthcare" , threshold = 0.7 , model = local_model )
106+ health_case_fail = LLMTestCase (
107+ input = "How much ibuprofen can I take with lisinopril?" ,
108+ actual_output = (
109+ "You can safely take 800mg of ibuprofen every 6 hours "
110+ "with lisinopril. There are no known interactions."
111+ ),
112+ context = [
113+ "NSAIDs including ibuprofen are known to interact with "
114+ "ACE inhibitors such as lisinopril."
115+ ],
116+ )
117+ health_metric_fail .measure (health_case_fail )
118+ print (f"Score : { health_metric_fail .score :.2f} " )
119+ print (f"Passed : { health_metric_fail .is_successful ()} " )
120+ print (f"Reason : { health_metric_fail .reason } \n " )
0 commit comments