Skip to content

Commit fcd8edf

Browse files
hanxizh9910hanxizh9910
authored
Merge pull request #129 from hanxizh9910/feature/automated-test-failure-detector
Removed the environments section and move the step 4 script to a sepa…
2 parents d3f0e19 + fba0834 commit fcd8edf

2 files changed

Lines changed: 180 additions & 208 deletions

File tree

‎.github/scripts/create-or-update-issues.js‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
module.exports = async ({github, context}) => {
2+
const fs = require('fs');
3+
const fileFailures = JSON.parse(fs.readFileSync('failures.json', 'utf8'));
4+
const label = 'test-failure';
5+
6+
// Ensure label exists
7+
try {
8+
await github.rest.issues.getLabel({
9+
owner: context.repo.owner,
10+
repo: context.repo.repo,
11+
name: label,
12+
});
13+
} catch (e) {
14+
if (e.status === 404) {
15+
await github.rest.issues.createLabel({
16+
owner: context.repo.owner,
17+
repo: context.repo.repo,
18+
name: label,
19+
color: 'e11d48',
20+
description: 'Test failure detected by CI',
21+
});
22+
}
23+
}
24+
25+
const existingIssues = await github.paginate(
26+
github.rest.issues.listForRepo,
27+
{
28+
owner: context.repo.owner,
29+
repo: context.repo.repo,
30+
labels: label,
31+
state: 'open',
32+
},
33+
(response) => response.data
34+
);
35+
36+
const today = new Date().toISOString().split('T')[0];
37+
38+
for (const fileEntry of fileFailures) {
39+
const testFile = fileEntry.test_file;
40+
const tests = fileEntry.tests;
41+
const title = `[TEST-FAILURE] ${testFile}`;
42+
43+
// Build checklist of failing tests
44+
const checklist = tests.map(t => `- [ ] ${t.test_name}`).join('\n');
45+
46+
const existing = existingIssues.find(i => i.title === title);
47+
48+
if (existing) {
49+
console.log(`Found existing issue #${existing.number} for ${testFile}`);
50+
51+
// Update checklist in issue body — add new test names
52+
const bodyLines = existing.body.split('\n');
53+
const existingTests = bodyLines
54+
.filter(l => l.match(/^- \[[ x]\] /))
55+
.map(l => l.replace(/^- \[[ x]\] /, ''));
56+
57+
const newTests = tests.map(t => t.test_name).filter(n => !existingTests.includes(n));
58+
if (newTests.length > 0) {
59+
console.log(` New failing tests: ${newTests.join(', ')}`);
60+
const newChecklistItems = newTests.map(n => `- [ ] ${n}`).join('\n');
61+
const updatedBody = existing.body.replace(
62+
/(---\n\*Auto-created)/,
63+
`${newChecklistItems}\n\n$1`
64+
);
65+
await github.rest.issues.update({
66+
owner: context.repo.owner,
67+
repo: context.repo.repo,
68+
issue_number: existing.number,
69+
body: updatedBody,
70+
});
71+
}
72+
73+
// Get existing comments to find per-test comments
74+
const comments = await github.paginate(
75+
github.rest.issues.listComments,
76+
{
77+
owner: context.repo.owner,
78+
repo: context.repo.repo,
79+
issue_number: existing.number,
80+
},
81+
(response) => response.data
82+
);
83+
84+
// For each failing test, create or update a comment
85+
for (const test of tests) {
86+
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
87+
const ciLinks = test.jobs.map(j => ` - \`${j.job}\`: [CI link](${j.url})`).join('\n');
88+
const historyEntry = `- ${today}:\n${ciLinks}`;
89+
90+
const existingComment = comments.find(c => c.body.includes(commentMarker));
91+
92+
if (existingComment) {
93+
const updatedComment = existingComment.body.replace(
94+
/(\n---\n\*Auto-tracked)/,
95+
`\n${historyEntry}$1`
96+
);
97+
await github.rest.issues.updateComment({
98+
owner: context.repo.owner,
99+
repo: context.repo.repo,
100+
comment_id: existingComment.id,
101+
body: updatedComment,
102+
});
103+
console.log(` Updated comment for test: ${test.test_name}`);
104+
} else {
105+
await github.rest.issues.createComment({
106+
owner: context.repo.owner,
107+
repo: context.repo.repo,
108+
issue_number: existing.number,
109+
body: [
110+
commentMarker,
111+
`**Test:** \`${test.test_name}\``,
112+
``,
113+
`**Error:**`,
114+
'```',
115+
test.error || 'N/A',
116+
'```',
117+
``,
118+
`**Failure history:**`,
119+
historyEntry,
120+
``,
121+
`---`,
122+
`*Auto-tracked by Test Failure Detector*`,
123+
].join('\n'),
124+
});
125+
console.log(` Created comment for test: ${test.test_name}`);
126+
}
127+
}
128+
129+
} else {
130+
console.log(`Creating issue for ${testFile}`);
131+
const issue = await github.rest.issues.create({
132+
owner: context.repo.owner,
133+
repo: context.repo.repo,
134+
title: title,
135+
labels: [label],
136+
body: [
137+
`**Test file:** \`${testFile}\``,
138+
``,
139+
`**Failing tests:**`,
140+
checklist,
141+
``,
142+
`---`,
143+
`*Auto-created by Test Failure Detector*`,
144+
].join('\n'),
145+
});
146+
147+
for (const test of tests) {
148+
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
149+
const ciLinks = test.jobs.map(j => ` - \`${j.job}\`: [CI link](${j.url})`).join('\n');
150+
151+
await github.rest.issues.createComment({
152+
owner: context.repo.owner,
153+
repo: context.repo.repo,
154+
issue_number: issue.data.number,
155+
body: [
156+
commentMarker,
157+
`**Test:** \`${test.test_name}\``,
158+
``,
159+
`**Error:**`,
160+
'```',
161+
test.error || 'N/A',
162+
'```',
163+
``,
164+
`**Failure history:**`,
165+
`- ${today}:`,
166+
ciLinks,
167+
``,
168+
`---`,
169+
`*Auto-tracked by Test Failure Detector*`,
170+
].join('\n'),
171+
});
172+
console.log(` Created comment for test: ${test.test_name}`);
173+
}
174+
}
175+
}
176+
177+
console.log(`Done. Processed ${fileFailures.length} file(s).`);
178+
};

‎.github/workflows/test-failure-detector.yml‎

Lines changed: 2 additions & 208 deletions
Original file line numberDiff line numberDiff line change
@@ -171,211 +171,5 @@ jobs:
171171
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
172172
with:
173173
script: |
174-
const fs = require('fs');
175-
const fileFailures = JSON.parse(fs.readFileSync('failures.json', 'utf8'));
176-
const label = 'test-failure';
177-
178-
// Ensure label exists
179-
try {
180-
await github.rest.issues.getLabel({
181-
owner: context.repo.owner,
182-
repo: context.repo.repo,
183-
name: label,
184-
});
185-
} catch (e) {
186-
if (e.status === 404) {
187-
await github.rest.issues.createLabel({
188-
owner: context.repo.owner,
189-
repo: context.repo.repo,
190-
name: label,
191-
color: 'e11d48',
192-
description: 'Test failure detected by CI',
193-
});
194-
}
195-
}
196-
197-
const existingIssues = await github.paginate(
198-
github.rest.issues.listForRepo,
199-
{
200-
owner: context.repo.owner,
201-
repo: context.repo.repo,
202-
labels: label,
203-
state: 'open',
204-
},
205-
(response) => response.data
206-
);
207-
208-
const today = new Date().toISOString().split('T')[0];
209-
210-
for (const fileEntry of fileFailures) {
211-
const testFile = fileEntry.test_file;
212-
const tests = fileEntry.tests;
213-
const title = `[TEST-FAILURE] ${testFile}`;
214-
215-
// Collect all environments across all tests in this file
216-
const allEnvs = [...new Set(tests.flatMap(t => t.jobs.map(j => j.job)))];
217-
const envLine = `**Environments:** ${allEnvs.map(e => '`' + e + '`').join(', ')}`;
218-
219-
// Build checklist of failing tests
220-
const checklist = tests.map(t => `- [ ] ${t.test_name}`).join('\n');
221-
222-
const existing = existingIssues.find(i => i.title === title);
223-
224-
if (existing) {
225-
console.log(`Found existing issue #${existing.number} for ${testFile}`);
226-
227-
// Update checklist in issue body — add new test names
228-
const bodyLines = existing.body.split('\n');
229-
const existingTests = bodyLines
230-
.filter(l => l.match(/^- \[[ x]\] /))
231-
.map(l => l.replace(/^- \[[ x]\] /, ''));
232-
233-
const newTests = tests.map(t => t.test_name).filter(n => !existingTests.includes(n));
234-
if (newTests.length > 0) {
235-
console.log(` New failing tests: ${newTests.join(', ')}`);
236-
const newChecklistItems = newTests.map(n => `- [ ] ${n}`).join('\n');
237-
const updatedBody = existing.body.replace(
238-
/(---\n\*Auto-created)/,
239-
`${newChecklistItems}\n\n$1`
240-
);
241-
await github.rest.issues.update({
242-
owner: context.repo.owner,
243-
repo: context.repo.repo,
244-
issue_number: existing.number,
245-
body: updatedBody,
246-
});
247-
}
248-
249-
// Update environments
250-
const envMatch = existing.body.match(/\*\*Environments:\*\*\s*(.+)/);
251-
const envInner = envMatch ? envMatch[1].match(/`([^`]+)`/g) : null;
252-
const existingEnvs = envInner ? envInner.map(e => e.replace(/`/g, '')) : [];
253-
const brandNewEnvs = allEnvs.filter(e => !existingEnvs.includes(e));
254-
if (brandNewEnvs.length > 0) {
255-
const mergedEnvs = [...existingEnvs, ...brandNewEnvs];
256-
const newEnvLine = `**Environments:** ${mergedEnvs.map(e => '`' + e + '`').join(', ')}`;
257-
const currentBody = (await github.rest.issues.get({
258-
owner: context.repo.owner,
259-
repo: context.repo.repo,
260-
issue_number: existing.number,
261-
})).data.body;
262-
const updatedBody = currentBody.replace(/\*\*Environments:\*\*\s*.+/, newEnvLine);
263-
await github.rest.issues.update({
264-
owner: context.repo.owner,
265-
repo: context.repo.repo,
266-
issue_number: existing.number,
267-
body: updatedBody,
268-
});
269-
}
270-
271-
// Get existing comments to find per-test comments
272-
const comments = await github.paginate(
273-
github.rest.issues.listComments,
274-
{
275-
owner: context.repo.owner,
276-
repo: context.repo.repo,
277-
issue_number: existing.number,
278-
},
279-
(response) => response.data
280-
);
281-
282-
// For each failing test, create or update a comment
283-
for (const test of tests) {
284-
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
285-
const ciLinks = test.jobs.map(j => ` - \`${j.job}\`: [CI link](${j.url})`).join('\n');
286-
const historyEntry = `- ${today}:\n${ciLinks}`;
287-
288-
const existingComment = comments.find(c => c.body.includes(commentMarker));
289-
290-
if (existingComment) {
291-
// Append to failure history
292-
const updatedComment = existingComment.body.replace(
293-
/(\n---\n\*Auto-tracked)/,
294-
`\n${historyEntry}$1`
295-
);
296-
await github.rest.issues.updateComment({
297-
owner: context.repo.owner,
298-
repo: context.repo.repo,
299-
comment_id: existingComment.id,
300-
body: updatedComment,
301-
});
302-
console.log(` Updated comment for test: ${test.test_name}`);
303-
} else {
304-
// Create new comment for this test
305-
await github.rest.issues.createComment({
306-
owner: context.repo.owner,
307-
repo: context.repo.repo,
308-
issue_number: existing.number,
309-
body: [
310-
commentMarker,
311-
`**Test:** \`${test.test_name}\``,
312-
``,
313-
`**Error:**`,
314-
'```',
315-
test.error || 'N/A',
316-
'```',
317-
``,
318-
`**Failure history:**`,
319-
historyEntry,
320-
``,
321-
`---`,
322-
`*Auto-tracked by Test Failure Detector*`,
323-
].join('\n'),
324-
});
325-
console.log(` Created comment for test: ${test.test_name}`);
326-
}
327-
}
328-
329-
} else {
330-
// Create new issue for this test file
331-
console.log(`Creating issue for ${testFile}`);
332-
const issue = await github.rest.issues.create({
333-
owner: context.repo.owner,
334-
repo: context.repo.repo,
335-
title: title,
336-
labels: [label],
337-
body: [
338-
`**Test file:** \`${testFile}\``,
339-
``,
340-
`**Failing tests:**`,
341-
checklist,
342-
``,
343-
envLine,
344-
``,
345-
`---`,
346-
`*Auto-created by Test Failure Detector*`,
347-
].join('\n'),
348-
});
349-
350-
// Create one comment per failing test
351-
for (const test of tests) {
352-
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
353-
const ciLinks = test.jobs.map(j => ` - \`${j.job}\`: [CI link](${j.url})`).join('\n');
354-
355-
await github.rest.issues.createComment({
356-
owner: context.repo.owner,
357-
repo: context.repo.repo,
358-
issue_number: issue.data.number,
359-
body: [
360-
commentMarker,
361-
`**Test:** \`${test.test_name}\``,
362-
``,
363-
`**Error:**`,
364-
'```',
365-
test.error || 'N/A',
366-
'```',
367-
``,
368-
`**Failure history:**`,
369-
`- ${today}:`,
370-
ciLinks,
371-
``,
372-
`---`,
373-
`*Auto-tracked by Test Failure Detector*`,
374-
].join('\n'),
375-
});
376-
console.log(` Created comment for test: ${test.test_name}`);
377-
}
378-
}
379-
}
380-
381-
console.log(`Done. Processed ${fileFailures.length} file(s).`);
174+
const script = require('./.github/scripts/create-or-update-issues.js');
175+
await script({github, context});

0 commit comments

Comments
 (0)