Skip to content

Commit bda98d5

Browse files
hanxizh9910hanxizh9910
authored
Merge branch 'unstable' into feature/automated-test-failure-detector
Signed-off-by: Hanxi Zhang <hanxizh@amazon.com>
2 parents 10048fb + e864e09 commit bda98d5

15 files changed

Lines changed: 736 additions & 243 deletions

File tree

‎.github/scripts/create-or-update-issues.js‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
module.exports = async ({github, context}) => {
2+
const fs = require('fs');
3+
const fileFailures = JSON.parse(fs.readFileSync('failures.json', 'utf8'));
4+
const label = 'test-failure';
5+
6+
// Ensure label exists
7+
try {
8+
await github.rest.issues.getLabel({
9+
owner: context.repo.owner,
10+
repo: context.repo.repo,
11+
name: label,
12+
});
13+
} catch (e) {
14+
if (e.status === 404) {
15+
await github.rest.issues.createLabel({
16+
owner: context.repo.owner,
17+
repo: context.repo.repo,
18+
name: label,
19+
color: 'e11d48',
20+
description: 'Test failure detected by CI',
21+
});
22+
}
23+
}
24+
25+
const existingIssues = await github.paginate(
26+
github.rest.issues.listForRepo,
27+
{
28+
owner: context.repo.owner,
29+
repo: context.repo.repo,
30+
labels: label,
31+
state: 'open',
32+
},
33+
(response) => response.data
34+
);
35+
36+
const today = new Date().toISOString().split('T')[0];
37+
38+
for (const fileEntry of fileFailures) {
39+
const testFile = fileEntry.test_file;
40+
const tests = fileEntry.tests;
41+
const title = `[TEST-FAILURE] ${testFile}`;
42+
43+
// Build checklist of failing tests
44+
const checklist = tests.map(t => `- [ ] ${t.test_name}`).join('\n');
45+
46+
const existing = existingIssues.find(i => i.title === title);
47+
48+
if (existing) {
49+
console.log(`Found existing issue #${existing.number} for ${testFile}`);
50+
51+
// Update checklist in issue body — add new test names
52+
const bodyLines = existing.body.split('\n');
53+
const existingTests = bodyLines
54+
.filter(l => l.match(/^- \[[ x]\] /))
55+
.map(l => l.replace(/^- \[[ x]\] /, ''));
56+
57+
const newTests = tests.map(t => t.test_name).filter(n => !existingTests.includes(n));
58+
if (newTests.length > 0) {
59+
console.log(` New failing tests: ${newTests.join(', ')}`);
60+
const newChecklistItems = newTests.map(n => `- [ ] ${n}`).join('\n');
61+
const updatedBody = existing.body.replace(
62+
/(---\n\*Auto-created)/,
63+
`${newChecklistItems}\n\n$1`
64+
);
65+
await github.rest.issues.update({
66+
owner: context.repo.owner,
67+
repo: context.repo.repo,
68+
issue_number: existing.number,
69+
body: updatedBody,
70+
});
71+
}
72+
73+
// Get existing comments to find per-test comments
74+
const comments = await github.paginate(
75+
github.rest.issues.listComments,
76+
{
77+
owner: context.repo.owner,
78+
repo: context.repo.repo,
79+
issue_number: existing.number,
80+
},
81+
(response) => response.data
82+
);
83+
84+
// For each failing test, create or update a comment
85+
for (const test of tests) {
86+
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
87+
const ciLinks = test.jobs.map(j => `- \`${j.job}\`: [CI link](${j.url})`).join('\n');
88+
const envNames = test.jobs.map(j => j.job);
89+
90+
const existingComment = comments.find(c => c.body.includes(commentMarker));
91+
92+
if (existingComment) {
93+
// Parse existing values
94+
const occMatch = existingComment.body.match(/\*\*Occurrences:\*\*\s*(\d+)/);
95+
const occurrences = occMatch ? parseInt(occMatch[1]) + 1 : 2;
96+
97+
const envMatch = existingComment.body.match(/\*\*Affected environments:\*\*\s*(.+)/);
98+
const envInner = envMatch ? envMatch[1].match(/`([^`]+)`/g) : null;
99+
const existingEnvs = envInner ? envInner.map(e => e.replace(/`/g, '')) : [];
100+
const allEnvs = [...new Set([...existingEnvs, ...envNames])];
101+
102+
// Keep first seen and first CI unchanged — extract them
103+
const firstSeenMatch = existingComment.body.match(/\*\*First seen:\*\*\s*(.+)/);
104+
const firstSeen = firstSeenMatch ? firstSeenMatch[1].trim() : today;
105+
106+
const firstCIMatch = existingComment.body.match(/\*\*First CI:\*\*\n([\s\S]*?)\n\n\*\*Last seen:/);
107+
const firstCI = firstCIMatch ? firstCIMatch[1].trim() : ciLinks;
108+
109+
const updatedComment = [
110+
commentMarker,
111+
`**Test:** \`${test.test_name}\``,
112+
``,
113+
`**Error:**`,
114+
'```',
115+
test.error || 'N/A',
116+
'```',
117+
``,
118+
`**First seen:** ${firstSeen}`,
119+
``,
120+
`**First CI:**`,
121+
firstCI,
122+
``,
123+
`**Last seen:** ${today}`,
124+
`**Occurrences:** ${occurrences}`,
125+
`**Affected environments:** ${allEnvs.map(e => '`' + e + '`').join(', ')}`,
126+
``,
127+
`**Latest CI:**`,
128+
ciLinks,
129+
``,
130+
`---`,
131+
`*Auto-tracked by Test Failure Detector*`,
132+
].join('\n');
133+
134+
await github.rest.issues.updateComment({
135+
owner: context.repo.owner,
136+
repo: context.repo.repo,
137+
comment_id: existingComment.id,
138+
body: updatedComment,
139+
});
140+
console.log(` Updated comment for test: ${test.test_name}`);
141+
} else {
142+
// Create new comment for this test
143+
await github.rest.issues.createComment({
144+
owner: context.repo.owner,
145+
repo: context.repo.repo,
146+
issue_number: existing.number,
147+
body: [
148+
commentMarker,
149+
`**Test:** \`${test.test_name}\``,
150+
``,
151+
`**Error:**`,
152+
'```',
153+
test.error || 'N/A',
154+
'```',
155+
``,
156+
`**First seen:** ${today}`,
157+
``,
158+
`**First CI:**`,
159+
ciLinks,
160+
``,
161+
`**Last seen:** ${today}`,
162+
`**Occurrences:** 1`,
163+
`**Affected environments:** ${envNames.map(e => '`' + e + '`').join(', ')}`,
164+
``,
165+
`**Latest CI:**`,
166+
ciLinks,
167+
``,
168+
`---`,
169+
`*Auto-tracked by Test Failure Detector*`,
170+
].join('\n'),
171+
});
172+
console.log(` Created comment for test: ${test.test_name}`);
173+
}
174+
}
175+
176+
} else {
177+
// Create new issue for this test file
178+
console.log(`Creating issue for ${testFile}`);
179+
const issue = await github.rest.issues.create({
180+
owner: context.repo.owner,
181+
repo: context.repo.repo,
182+
title: title,
183+
labels: [label],
184+
body: [
185+
`**Test file:** \`${testFile}\``,
186+
``,
187+
`**Failing tests:**`,
188+
checklist,
189+
``,
190+
`---`,
191+
`*Auto-created by Test Failure Detector*`,
192+
].join('\n'),
193+
});
194+
195+
// Small delay to allow GitHub to fully process the new issue
196+
await new Promise(resolve => setTimeout(resolve, 2000));
197+
198+
// Create one comment per failing test
199+
for (const test of tests) {
200+
const commentMarker = `<!-- test-failure: ${test.test_name} -->`;
201+
const ciLinks = test.jobs.map(j => `- \`${j.job}\`: [CI link](${j.url})`).join('\n');
202+
const envNames = test.jobs.map(j => j.job);
203+
204+
await github.rest.issues.createComment({
205+
owner: context.repo.owner,
206+
repo: context.repo.repo,
207+
issue_number: issue.data.number,
208+
body: [
209+
commentMarker,
210+
`**Test:** \`${test.test_name}\``,
211+
``,
212+
`**Error:**`,
213+
'```',
214+
test.error || 'N/A',
215+
'```',
216+
``,
217+
`**First seen:** ${today}`,
218+
``,
219+
`**First CI:**`,
220+
ciLinks,
221+
``,
222+
`**Last seen:** ${today}`,
223+
`**Occurrences:** 1`,
224+
`**Affected environments:** ${envNames.map(e => '`' + e + '`').join(', ')}`,
225+
``,
226+
`**Latest CI:**`,
227+
ciLinks,
228+
``,
229+
`---`,
230+
`*Auto-tracked by Test Failure Detector*`,
231+
].join('\n'),
232+
});
233+
console.log(` Created comment for test: ${test.test_name}`);
234+
}
235+
}
236+
}
237+
238+
console.log(`Done. Processed ${fileFailures.length} file(s).`);
239+
};

0 commit comments

Comments
 (0)