Skip to content

Commit ac58d5f

Browse files
authored
feat(argilla): add Argilla MCP Server with 13 tools (#13)
## Dataset Tools (8) - create_dataset: tạo dataset với fields & questions - push_records: đẩy records từ pipeline vào Argilla - get_records: lấy records đã annotate - query_records: query theo filter (status/annotator/label) - export_dataset: export ra JSONL hoặc Parquet - delete_records: xóa records theo filter - annotation_progress: thống kê tiến độ annotation - agreement_score: tính IAA (Cohen/Fleiss Kappa, Krippendorff Alpha, Overlap) ## User/Annotator Management Tools (5) - create_user: tạo user mới (admin | annotator) - list_users: liệt kê users theo role - delete_user: xóa user theo username - manage_workspace: thêm/xóa user khỏi workspace - annotator_stats: thống kê annotations theo annotator ## Factory Modules (SOLID) - connection.py: ArgillaConnectionFactory (Singleton) - dataset.py: DatasetFactory - record.py: RecordFactory - export.py: ExportFactory (JSONL/Parquet/HF Hub) - agreement.py: AgreementFactory (IAA algorithms) - user.py: UserFactory ## Dependencies - argilla>=2.0.0, scikit-learn>=1.3.0, krippendorff>=0.6.0
1 parent 421a637 commit ac58d5f

11 files changed

Lines changed: 1846 additions & 0 deletions

File tree

pyproject.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,13 @@ evaluator = [
122122
"litellm>=1.0.0",
123123
]
124124

125+
# ── Argilla: Data Annotation Platform ─────────────────────────────────────────
126+
argilla = [
127+
"argilla>=2.0.0",
128+
"scikit-learn>=1.3.0",
129+
"krippendorff>=0.6.0",
130+
]
131+
125132
# evaluator-local: chạy test/eval model local (HuggingFace)
126133
evaluator-local = [
127134
"opik>=1.10.9",
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Argilla MCP Server for Zem
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Argilla factory modules
Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
"""
2+
AgreementFactory – tính độ đồng thuận giữa các annotators (Inter-Annotator Agreement).
3+
4+
Hỗ trợ:
5+
- Cohen's Kappa : 2 annotators, categorical
6+
- Fleiss' Kappa : N annotators, categorical
7+
- Krippendorff Alpha : N annotators, mọi thang đo
8+
- Overlap % : % records được ít nhất K annotators label
9+
- Label Distribution : phân phối nhãn
10+
"""
11+
from typing import Any, Dict, List, Optional, Tuple
12+
from loguru import logger
13+
14+
15+
class AgreementFactory:
16+
"""
17+
Factory tính các chỉ số đồng thuận Inter-Annotator Agreement (IAA).
18+
"""
19+
20+
# ── Diễn giải Kappa / Alpha ─────────────────────────────────────────────
21+
22+
@staticmethod
23+
def _interpret_kappa(score: float) -> str:
24+
"""Diễn giải điểm Kappa theo Landis & Koch (1977)."""
25+
if score < 0:
26+
return "Kém (< 0: đồng thuận tệ hơn ngẫu nhiên)"
27+
elif score < 0.20:
28+
return "Không đáng kể (0–0.20)"
29+
elif score < 0.40:
30+
return "Yếu (0.20–0.40)"
31+
elif score < 0.60:
32+
return "Trung bình (0.40–0.60)"
33+
elif score < 0.80:
34+
return "Khá tốt (0.60–0.80)"
35+
else:
36+
return "Xuất sắc (0.80–1.00)"
37+
38+
# ── Cohen's Kappa ────────────────────────────────────────────────────────
39+
40+
@staticmethod
41+
def cohen_kappa(
42+
labels_a: List[Any],
43+
labels_b: List[Any],
44+
) -> Dict[str, Any]:
45+
"""
46+
Tính Cohen's Kappa giữa 2 annotators.
47+
48+
Args:
49+
labels_a: Danh sách nhãn của annotator A
50+
labels_b: Danh sách nhãn của annotator B
51+
52+
Returns:
53+
dict với score, method, n_samples, interpretation
54+
"""
55+
try:
56+
from sklearn.metrics import cohen_kappa_score
57+
except ImportError:
58+
raise ImportError("Cài: pip install 'xfmr-zem[argilla]'")
59+
60+
if len(labels_a) != len(labels_b):
61+
raise ValueError(
62+
f"Số nhãn không khớp: annotator_a={len(labels_a)}, annotator_b={len(labels_b)}"
63+
)
64+
65+
score = float(cohen_kappa_score(labels_a, labels_b))
66+
result = {
67+
"method": "cohen_kappa",
68+
"score": round(score, 4),
69+
"n_samples": len(labels_a),
70+
"interpretation": AgreementFactory._interpret_kappa(score),
71+
}
72+
logger.info(f"Cohen's Kappa = {score:.4f} ({result['interpretation']})")
73+
return result
74+
75+
# ── Fleiss' Kappa ────────────────────────────────────────────────────────
76+
77+
@staticmethod
78+
def fleiss_kappa(
79+
ratings_matrix: List[List[int]],
80+
labels: Optional[List[Any]] = None,
81+
) -> Dict[str, Any]:
82+
"""
83+
Tính Fleiss' Kappa cho N annotators.
84+
85+
Args:
86+
ratings_matrix: Ma trận ratings.
87+
Mỗi hàng = 1 record, mỗi cột = số annotators gán nhãn đó.
88+
Hoặc có thể là list of lists [annotator][record] = label.
89+
labels: Danh sách nhãn (dùng khi ratings_matrix là dạng list nhãn)
90+
91+
Returns:
92+
dict với score, method, n_samples, n_annotators, interpretation
93+
"""
94+
import numpy as np
95+
96+
n_records = len(ratings_matrix)
97+
matrix = np.array(ratings_matrix, dtype=float)
98+
99+
# Kiểm tra có dạng count matrix không
100+
if matrix.ndim == 2:
101+
n_annotators = int(matrix.sum(axis=1).mean())
102+
n_categories = matrix.shape[1]
103+
else:
104+
raise ValueError("ratings_matrix phải là 2D: [n_records × n_categories]")
105+
106+
# Fleiss' Kappa formula
107+
p_j = matrix.sum(axis=0) / (n_records * n_annotators) # proportion mỗi category
108+
P_e = float(np.sum(p_j ** 2))
109+
110+
P_i = (np.sum(matrix ** 2, axis=1) - n_annotators) / (n_annotators * (n_annotators - 1))
111+
P_bar = float(np.mean(P_i))
112+
113+
if P_e == 1.0:
114+
kappa = 0.0
115+
else:
116+
kappa = (P_bar - P_e) / (1 - P_e)
117+
118+
result = {
119+
"method": "fleiss_kappa",
120+
"score": round(kappa, 4),
121+
"n_samples": n_records,
122+
"n_categories": n_categories,
123+
"n_annotators": n_annotators,
124+
"interpretation": AgreementFactory._interpret_kappa(kappa),
125+
}
126+
logger.info(f"Fleiss' Kappa = {kappa:.4f} ({result['interpretation']})")
127+
return result
128+
129+
# ── Krippendorff Alpha ───────────────────────────────────────────────────
130+
131+
@staticmethod
132+
def krippendorff_alpha(
133+
reliability_data: List[List[Optional[Any]]],
134+
data_type: str = "nominal",
135+
) -> Dict[str, Any]:
136+
"""
137+
Tính Krippendorff Alpha cho N annotators với mọi thang đo.
138+
139+
Args:
140+
reliability_data: Ma trận [n_annotators × n_records].
141+
Dùng None cho missing data.
142+
data_type: "nominal" | "ordinal" | "interval" | "ratio"
143+
144+
Returns:
145+
dict với score, method, data_type, interpretation
146+
"""
147+
try:
148+
import krippendorff
149+
except ImportError:
150+
raise ImportError("Cài: pip install krippendorff")
151+
152+
import numpy as np
153+
154+
# Chuyển None → np.nan
155+
data = np.array(
156+
[[np.nan if v is None else v for v in row] for row in reliability_data],
157+
dtype=float,
158+
)
159+
160+
alpha = float(krippendorff.alpha(reliability_data=data, level_of_measurement=data_type))
161+
162+
result = {
163+
"method": "krippendorff_alpha",
164+
"score": round(alpha, 4),
165+
"data_type": data_type,
166+
"n_annotators": data.shape[0],
167+
"n_samples": data.shape[1],
168+
"interpretation": AgreementFactory._interpret_kappa(alpha),
169+
}
170+
logger.info(f"Krippendorff Alpha = {alpha:.4f} (type={data_type})")
171+
return result
172+
173+
# ── Annotation Overlap % ─────────────────────────────────────────────────
174+
175+
@staticmethod
176+
def overlap_percentage(
177+
records: List[Any],
178+
min_annotators: int = 2,
179+
) -> Dict[str, Any]:
180+
"""
181+
Tính % records được ít nhất `min_annotators` người label.
182+
183+
Args:
184+
records: List rg.Record từ Argilla
185+
min_annotators: Ngưỡng tối thiểu
186+
187+
Returns:
188+
dict với overlap_pct, total_records, covered_records, min_annotators
189+
"""
190+
total = len(records)
191+
covered = sum(
192+
1 for rec in records
193+
if rec.responses and len(rec.responses) >= min_annotators
194+
)
195+
pct = round(covered / total * 100, 2) if total > 0 else 0.0
196+
197+
result = {
198+
"method": "overlap",
199+
"total_records": total,
200+
"covered_records": covered,
201+
"min_annotators": min_annotators,
202+
"overlap_pct": pct,
203+
}
204+
logger.info(f"Overlap = {pct}% ({covered}/{total} records với ≥{min_annotators} annotators)")
205+
return result
206+
207+
# ── Label Distribution ───────────────────────────────────────────────────
208+
209+
@staticmethod
210+
def label_distribution(
211+
records: List[Any],
212+
question_name: str,
213+
) -> Dict[str, Any]:
214+
"""
215+
Tính phân phối nhãn trên một question.
216+
217+
Args:
218+
records: List rg.Record từ Argilla
219+
question_name: Tên question cần phân tích
220+
221+
Returns:
222+
dict với distribution (label → count), total_annotations, per_annotator
223+
"""
224+
from collections import Counter, defaultdict
225+
226+
distribution: Counter = Counter()
227+
per_annotator: Dict[str, Counter] = defaultdict(Counter)
228+
total = 0
229+
230+
for rec in records:
231+
if not rec.responses:
232+
continue
233+
for resp in rec.responses:
234+
if not resp.values or question_name not in resp.values:
235+
continue
236+
val = resp.values[question_name].value
237+
# Handle list (multi-label)
238+
labels = val if isinstance(val, list) else [val]
239+
for label in labels:
240+
distribution[str(label)] += 1
241+
annotator = str(resp.user_id) if resp.user_id else "unknown"
242+
per_annotator[annotator][str(label)] += 1
243+
total += 1
244+
245+
result = {
246+
"method": "label_distribution",
247+
"question_name": question_name,
248+
"total_annotations": total,
249+
"distribution": dict(distribution),
250+
"per_annotator": {k: dict(v) for k, v in per_annotator.items()},
251+
}
252+
logger.info(f"Label distribution cho '{question_name}': {dict(distribution)}")
253+
return result
254+
255+
# ── Dispatcher ───────────────────────────────────────────────────────────
256+
257+
@classmethod
258+
def compute(
259+
cls,
260+
method: str,
261+
records: Optional[List[Any]] = None,
262+
question_name: str = "label",
263+
data_type: str = "nominal",
264+
min_annotators: int = 2,
265+
) -> Dict[str, Any]:
266+
"""
267+
Dispatcher tổng quát. Tự động extract labels từ records và tính IAA.
268+
269+
Args:
270+
method: "cohen_kappa" | "fleiss_kappa" | "krippendorff" | "overlap" | "distribution"
271+
records: List rg.Record từ Argilla
272+
question_name: Tên question để lấy nhãn
273+
data_type: Thang đo cho krippendorff
274+
min_annotators: Cho overlap
275+
276+
Returns:
277+
dict kết quả IAA
278+
"""
279+
if method == "overlap":
280+
return cls.overlap_percentage(records, min_annotators)
281+
282+
if method == "distribution":
283+
return cls.label_distribution(records, question_name)
284+
285+
# Thu thập annotations theo annotator
286+
from collections import defaultdict
287+
annotator_labels: Dict[str, Dict[int, Any]] = defaultdict(dict)
288+
289+
for i, rec in enumerate(records):
290+
if not rec.responses:
291+
continue
292+
for resp in rec.responses:
293+
if not resp.values or question_name not in resp.values:
294+
continue
295+
annotator = str(resp.user_id) if resp.user_id else f"anon_{i}"
296+
annotator_labels[annotator][i] = resp.values[question_name].value
297+
298+
annotators = list(annotator_labels.keys())
299+
n_records = len(records)
300+
301+
if method == "cohen_kappa":
302+
if len(annotators) < 2:
303+
raise ValueError(f"Cohen's Kappa cần ≥2 annotators, có {len(annotators)}")
304+
a1 = annotators[0]
305+
a2 = annotators[1]
306+
# Chỉ lấy records cả 2 cùng annotate
307+
common_ids = sorted(
308+
set(annotator_labels[a1].keys()) & set(annotator_labels[a2].keys())
309+
)
310+
if not common_ids:
311+
raise ValueError("Không có records nào được cả 2 annotators label")
312+
la = [annotator_labels[a1][i] for i in common_ids]
313+
lb = [annotator_labels[a2][i] for i in common_ids]
314+
return cls.cohen_kappa(la, lb)
315+
316+
elif method == "fleiss_kappa":
317+
# Xây count matrix
318+
import numpy as np
319+
all_labels = sorted(set(
320+
v for ann_dict in annotator_labels.values()
321+
for v in ann_dict.values()
322+
))
323+
label_idx = {l: i for i, l in enumerate(all_labels)}
324+
matrix = np.zeros((n_records, len(all_labels)), dtype=float)
325+
for ann_labels in annotator_labels.values():
326+
for rec_idx, label in ann_labels.items():
327+
matrix[rec_idx, label_idx[label]] += 1
328+
return cls.fleiss_kappa(matrix.tolist())
329+
330+
elif method == "krippendorff":
331+
import numpy as np
332+
all_record_ids = list(range(n_records))
333+
rel_data = []
334+
for ann in annotators:
335+
row = [annotator_labels[ann].get(i, None) for i in all_record_ids]
336+
# Convert str labels to int nếu cần (nominal OK với str)
337+
rel_data.append(row)
338+
return cls.krippendorff_alpha(rel_data, data_type=data_type)
339+
340+
else:
341+
raise ValueError(
342+
f"Method không hỗ trợ: '{method}'. "
343+
"Dùng: cohen_kappa | fleiss_kappa | krippendorff | overlap | distribution"
344+
)

0 commit comments

Comments
 (0)