Skip to content

Commit d593974

Browse files
penguine-ippenguine-ip
authored
Merge pull request #2633 from confident-ai/features/synthesizer-CLI
added CLI for generate goldens
2 parents f0a08f5 + 7579232 commit d593974

12 files changed

Lines changed: 1203 additions & 44 deletions

File tree

‎deepeval/cli/generate/__init__.py‎

Whitespace-only changes.

‎deepeval/cli/generate/command.py‎

Lines changed: 309 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,309 @@
1+
from pathlib import Path
2+
from typing import List, Optional
3+
4+
import typer
5+
from rich import print
6+
7+
from deepeval.synthesizer import Synthesizer
8+
from deepeval.synthesizer.config import ContextConstructionConfig
9+
from deepeval.cli.generate.utils import (
10+
FileType,
11+
GenerationMethod,
12+
GoldenVariation,
13+
load_contexts_file,
14+
load_goldens_file,
15+
multi_turn_styling_config,
16+
require_method_option,
17+
single_turn_styling_config,
18+
validate_golden_variation,
19+
validate_scratch_styling,
20+
)
21+
22+
23+
def generate_command(
24+
method: GenerationMethod = typer.Option(
25+
...,
26+
"--method",
27+
help="Golden generation method to use.",
28+
case_sensitive=False,
29+
),
30+
variation: GoldenVariation = typer.Option(
31+
...,
32+
"--variation",
33+
help="Golden variation to generate.",
34+
case_sensitive=False,
35+
),
36+
output_dir: str = typer.Option(
37+
"./synthetic_data",
38+
"--output-dir",
39+
help="Directory where generated goldens will be saved.",
40+
),
41+
file_type: FileType = typer.Option(
42+
FileType.JSON,
43+
"--file-type",
44+
help="File type to save generated goldens as.",
45+
case_sensitive=False,
46+
),
47+
file_name: Optional[str] = typer.Option(
48+
None,
49+
"--file-name",
50+
help="Optional output filename without extension.",
51+
),
52+
model: Optional[str] = typer.Option(
53+
None,
54+
"--model",
55+
help="Model to use for generation.",
56+
),
57+
async_mode: bool = typer.Option(
58+
True,
59+
"--async-mode/--sync-mode",
60+
help="Whether to generate goldens concurrently.",
61+
),
62+
max_concurrent: int = typer.Option(
63+
100,
64+
"--max-concurrent",
65+
help="Maximum number of concurrent generation tasks.",
66+
),
67+
include_expected: bool = typer.Option(
68+
True,
69+
"--include-expected/--no-include-expected",
70+
help="Whether to generate expected output or expected outcome.",
71+
),
72+
cost_tracking: bool = typer.Option(
73+
False,
74+
"--cost-tracking",
75+
help="Print generation cost when supported by the model.",
76+
),
77+
documents: Optional[List[str]] = typer.Option(
78+
None,
79+
"--documents",
80+
help="Document path to use with --method docs. Can be passed multiple times.",
81+
),
82+
contexts_file: Optional[Path] = typer.Option(
83+
None,
84+
"--contexts-file",
85+
help='JSON file shaped like [["chunk 1", "chunk 2"], ...].',
86+
),
87+
goldens_file: Optional[Path] = typer.Option(
88+
None,
89+
"--goldens-file",
90+
help="Existing goldens file to augment (.json, .csv, or .jsonl).",
91+
),
92+
num_goldens: Optional[int] = typer.Option(
93+
None,
94+
"--num-goldens",
95+
help="Number of goldens to generate with --method scratch.",
96+
),
97+
max_goldens_per_context: int = typer.Option(
98+
2,
99+
"--max-goldens-per-context",
100+
help="Maximum goldens to generate per context.",
101+
),
102+
max_goldens_per_golden: int = typer.Option(
103+
2,
104+
"--max-goldens-per-golden",
105+
help="Maximum goldens to generate per existing golden.",
106+
),
107+
max_contexts_per_document: int = typer.Option(
108+
3,
109+
"--max-contexts-per-document",
110+
help="Maximum contexts to construct per document.",
111+
),
112+
min_contexts_per_document: int = typer.Option(
113+
1,
114+
"--min-contexts-per-document",
115+
help="Minimum contexts to construct per document.",
116+
),
117+
chunk_size: int = typer.Option(
118+
1024,
119+
"--chunk-size",
120+
help="Token chunk size for document parsing.",
121+
),
122+
chunk_overlap: int = typer.Option(
123+
0,
124+
"--chunk-overlap",
125+
help="Token overlap between document chunks.",
126+
),
127+
context_quality_threshold: float = typer.Option(
128+
0.5,
129+
"--context-quality-threshold",
130+
help="Minimum context quality threshold.",
131+
),
132+
context_similarity_threshold: float = typer.Option(
133+
0.0,
134+
"--context-similarity-threshold",
135+
help="Minimum context grouping similarity threshold.",
136+
),
137+
max_retries: int = typer.Option(
138+
3,
139+
"--max-retries",
140+
help="Maximum retries for context construction quality checks.",
141+
),
142+
scenario: Optional[str] = typer.Option(
143+
None,
144+
"--scenario",
145+
help="Single-turn generation scenario.",
146+
),
147+
task: Optional[str] = typer.Option(
148+
None,
149+
"--task",
150+
help="Single-turn generation task.",
151+
),
152+
input_format: Optional[str] = typer.Option(
153+
None,
154+
"--input-format",
155+
help="Single-turn input format.",
156+
),
157+
expected_output_format: Optional[str] = typer.Option(
158+
None,
159+
"--expected-output-format",
160+
help="Single-turn expected output format.",
161+
),
162+
scenario_context: Optional[str] = typer.Option(
163+
None,
164+
"--scenario-context",
165+
help="Multi-turn scenario context.",
166+
),
167+
conversational_task: Optional[str] = typer.Option(
168+
None,
169+
"--conversational-task",
170+
help="Multi-turn conversational task.",
171+
),
172+
participant_roles: Optional[str] = typer.Option(
173+
None,
174+
"--participant-roles",
175+
help="Multi-turn participant roles.",
176+
),
177+
scenario_format: Optional[str] = typer.Option(
178+
None,
179+
"--scenario-format",
180+
help="Multi-turn scenario format.",
181+
),
182+
expected_outcome_format: Optional[str] = typer.Option(
183+
None,
184+
"--expected-outcome-format",
185+
help="Multi-turn expected outcome format.",
186+
),
187+
):
188+
"""Generate synthetic goldens with the golden synthesizer."""
189+
document_paths = None
190+
contexts = None
191+
goldens = None
192+
193+
if method == GenerationMethod.DOCS:
194+
document_paths = require_method_option(
195+
documents, "--documents", method
196+
)
197+
elif method == GenerationMethod.CONTEXTS:
198+
contexts_path = require_method_option(
199+
contexts_file, "--contexts-file", method
200+
)
201+
contexts = load_contexts_file(contexts_path)
202+
elif method == GenerationMethod.SCRATCH:
203+
require_method_option(num_goldens, "--num-goldens", method)
204+
validate_scratch_styling(
205+
variation=variation,
206+
scenario=scenario,
207+
task=task,
208+
input_format=input_format,
209+
scenario_context=scenario_context,
210+
conversational_task=conversational_task,
211+
participant_roles=participant_roles,
212+
)
213+
elif method == GenerationMethod.GOLDENS:
214+
goldens_path = require_method_option(
215+
goldens_file, "--goldens-file", method
216+
)
217+
goldens = load_goldens_file(goldens_path)
218+
validate_golden_variation(goldens, variation)
219+
220+
styling_config = single_turn_styling_config(
221+
scenario=scenario,
222+
task=task,
223+
input_format=input_format,
224+
expected_output_format=expected_output_format,
225+
)
226+
conversational_styling_config = multi_turn_styling_config(
227+
scenario_context=scenario_context,
228+
conversational_task=conversational_task,
229+
participant_roles=participant_roles,
230+
scenario_format=scenario_format,
231+
expected_outcome_format=expected_outcome_format,
232+
)
233+
synthesizer = Synthesizer(
234+
model=model,
235+
async_mode=async_mode,
236+
max_concurrent=max_concurrent,
237+
styling_config=styling_config,
238+
conversational_styling_config=conversational_styling_config,
239+
cost_tracking=cost_tracking,
240+
)
241+
242+
if method == GenerationMethod.DOCS:
243+
context_construction_config = ContextConstructionConfig(
244+
max_contexts_per_document=max_contexts_per_document,
245+
min_contexts_per_document=min_contexts_per_document,
246+
chunk_size=chunk_size,
247+
chunk_overlap=chunk_overlap,
248+
context_quality_threshold=context_quality_threshold,
249+
context_similarity_threshold=context_similarity_threshold,
250+
max_retries=max_retries,
251+
)
252+
if variation == GoldenVariation.SINGLE_TURN:
253+
synthesizer.generate_goldens_from_docs(
254+
document_paths=document_paths,
255+
include_expected_output=include_expected,
256+
max_goldens_per_context=max_goldens_per_context,
257+
context_construction_config=context_construction_config,
258+
)
259+
else:
260+
synthesizer.generate_conversational_goldens_from_docs(
261+
document_paths=document_paths,
262+
include_expected_outcome=include_expected,
263+
max_goldens_per_context=max_goldens_per_context,
264+
context_construction_config=context_construction_config,
265+
)
266+
267+
elif method == GenerationMethod.CONTEXTS:
268+
if variation == GoldenVariation.SINGLE_TURN:
269+
synthesizer.generate_goldens_from_contexts(
270+
contexts=contexts,
271+
include_expected_output=include_expected,
272+
max_goldens_per_context=max_goldens_per_context,
273+
)
274+
else:
275+
synthesizer.generate_conversational_goldens_from_contexts(
276+
contexts=contexts,
277+
include_expected_outcome=include_expected,
278+
max_goldens_per_context=max_goldens_per_context,
279+
)
280+
281+
elif method == GenerationMethod.SCRATCH:
282+
if variation == GoldenVariation.SINGLE_TURN:
283+
synthesizer.generate_goldens_from_scratch(num_goldens=num_goldens)
284+
else:
285+
synthesizer.generate_conversational_goldens_from_scratch(
286+
num_goldens=num_goldens
287+
)
288+
289+
elif method == GenerationMethod.GOLDENS:
290+
if variation == GoldenVariation.SINGLE_TURN:
291+
synthesizer.generate_goldens_from_goldens(
292+
goldens=goldens,
293+
max_goldens_per_golden=max_goldens_per_golden,
294+
include_expected_output=include_expected,
295+
)
296+
else:
297+
synthesizer.generate_conversational_goldens_from_goldens(
298+
goldens=goldens,
299+
max_goldens_per_golden=max_goldens_per_golden,
300+
include_expected_outcome=include_expected,
301+
)
302+
303+
output_path = synthesizer.save_as(
304+
file_type=file_type.value,
305+
directory=output_dir,
306+
file_name=file_name,
307+
quiet=True,
308+
)
309+
print(f"Synthetic goldens saved at {output_path}!")

0 commit comments

Comments
 (0)