Skip to content

Commit 6873abd

Browse files
FrawlessFrawless
committed
Some fixes after rebase and adding tests
Signed-off-by: Jakub Stejskal <xstejs24@gmail.com>
1 parent 844575b commit 6873abd

12 files changed

Lines changed: 411 additions & 53 deletions

File tree

‎.github/actions/systemtests/parse-comment/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@ runs:
141141
};
142142
143143
const DEFAULTS = {
144-
pipelines: 'regression,upgrade',
145-
profiles: 'operators,operands,brokers-and-security,azp_kraft_upgrade,azp_kafka_upgrade',
144+
pipelines: '',
145+
profiles: '',
146146
featureGate: '+DummyFeatureGate',
147147
rbacScope: 'CLUSTER',
148148
installType: 'yaml',

‎.github/actions/systemtests/validate-matrix/action.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: "Validate Matrix"
2+
description: "Validates generated matrix and returns feedback message"
3+
4+
inputs:
5+
matrix:
6+
description: "JSON matrix generated by generate-matrix action"
7+
required: true
8+
9+
outputs:
10+
isValid:
11+
description: "Whether the matrix is valid (not empty)"
12+
value: ${{ steps.validate.outputs.isValid }}
13+
message:
14+
description: "Feedback message about the validation result"
15+
value: ${{ steps.validate.outputs.message }}
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Validate Matrix
21+
id: validate
22+
uses: actions/github-script@v7
23+
with:
24+
script: |
25+
const matrix = JSON.parse(process.env.MATRIX || '[]');
26+
27+
// Check if matrix is empty
28+
const isEmpty = !matrix || matrix.length === 0;
29+
30+
if (isEmpty) {
31+
core.setOutput('isValid', 'false');
32+
33+
let errorMessage = "❌ **System test validation failed**\n";
34+
errorMessage += "\n";
35+
errorMessage += "The provided pipeline/profile names did not match any configured tests.\n";
36+
errorMessage += "\n";
37+
errorMessage += "**Available pipelines:**\n";
38+
errorMessage += "`acceptance`, `acceptance-helm`, `regression`, `regression-fg`, `regression-rbac`, `smoke`, `upgrade`\n";
39+
errorMessage += "\n";
40+
errorMessage += "**Available profiles:**\n";
41+
errorMessage += "`acceptance`, `azp_connect_mirrormaker`, `azp_dynconfig_listeners_tracing_watcher`, `azp_kafka_oauth`, `azp_kafka_upgrade`, `azp_kraft_upgrade`, `azp_operators`, `azp_rbac_remaining`, `azp_rolling_update_bridge`, `azp_security`, `brokers-and-security`, `operands`, `operators`, `smoke`\n";
42+
errorMessage += "\n";
43+
errorMessage += "Please check your command for typos and try again.";
44+
45+
core.setOutput('message', errorMessage);
46+
return;
47+
}
48+
49+
// Matrix is valid - create summary of what will run
50+
core.setOutput('isValid', 'true');
51+
core.info(`Matrix validation passed. ${matrix.length} job(s) will be executed.`);
52+
53+
const jobs = matrix.map(job => {
54+
const name = job.jobName || `${job.pipeline}-${job.profile || job.pipeline}-${job.arch}`;
55+
const agent = job.agent || 'default';
56+
const timeout = job.timeout || 'default';
57+
return `- **${name}** (${agent}, ${timeout}min)`;
58+
}).join('\n');
59+
60+
let successMessage = "✅ **System test validation passed**\n";
61+
successMessage += "\n";
62+
successMessage += `The following ${matrix.length} job(s) will be executed:\n`;
63+
successMessage += "\n";
64+
successMessage += jobs.replace(/\n/g, '\n') + "\n";
65+
successMessage += "\n";
66+
successMessage += "Tests will start after successful build completion.";
67+
68+
core.setOutput('message', successMessage);
69+
env:
70+
MATRIX: ${{ inputs.matrix }}

‎.github/tests/inputs/validate-matrix/empty_matrix.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎.github/tests/inputs/validate-matrix/null_matrix.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
null

‎.github/tests/inputs/validate-matrix/valid_custom.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[
2+
{
3+
"agent": "oracle-4cpu-16gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "custom",
6+
"profile": "all",
7+
"timeout": 720,
8+
"strimzi_feature_gates": "",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "bundle",
11+
"parallel": "1",
12+
"groups": "",
13+
"tests": "",
14+
"jobName": "custom-all-amd64"
15+
},
16+
{
17+
"agent": "oracle-4cpu-16gb-arm64",
18+
"arch": "arm64",
19+
"pipeline": "custom",
20+
"profile": "all",
21+
"timeout": 720,
22+
"strimzi_feature_gates": "",
23+
"strimzi_rbac_scope": "CLUSTER",
24+
"cluster_operator_install_type": "bundle",
25+
"parallel": "1",
26+
"groups": "",
27+
"tests": "",
28+
"jobName": "custom-all-arm64"
29+
}
30+
]

‎.github/tests/inputs/validate-matrix/valid_multiple.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"agent": "oracle-vm-8cpu-32gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "regression",
6+
"profile": "brokers-and-security",
7+
"timeout": 480,
8+
"strimzi_feature_gates": "false",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "yaml",
11+
"parallel": 4,
12+
"jobName": "regression-brokers-and-security-amd64"
13+
},
14+
{
15+
"agent": "oracle-vm-8cpu-32gb-x86-64",
16+
"arch": "amd64",
17+
"pipeline": "regression",
18+
"profile": "operators",
19+
"timeout": 480,
20+
"strimzi_feature_gates": "false",
21+
"strimzi_rbac_scope": "CLUSTER",
22+
"cluster_operator_install_type": "yaml",
23+
"parallel": 4,
24+
"jobName": "regression-operators-amd64"
25+
},
26+
{
27+
"agent": "oracle-vm-2cpu-8gb-x86-64",
28+
"arch": "amd64",
29+
"pipeline": "smoke",
30+
"profile": "smoke",
31+
"timeout": 60,
32+
"strimzi_feature_gates": "false",
33+
"strimzi_rbac_scope": "CLUSTER",
34+
"cluster_operator_install_type": "yaml",
35+
"parallel": 1,
36+
"jobName": "smoke-amd64"
37+
}
38+
]

‎.github/tests/inputs/validate-matrix/valid_single.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[
2+
{
3+
"agent": "oracle-vm-2cpu-8gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "smoke",
6+
"profile": "smoke",
7+
"timeout": 60,
8+
"strimzi_feature_gates": "false",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "yaml",
11+
"parallel": 1,
12+
"jobName": "smoke-amd64"
13+
}
14+
]

‎.github/tests/scenarios/parse-comment.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ scenarios:
3939
fixture: .github/tests/events/parse-comment/issue_comment_3.json
4040
expectations:
4141
pipelines: ""
42-
profiles: "operators,operands,brokers-and-security,azp_kraft_upgrade,azp_kafka_upgrade"
42+
profiles: ""
4343
tests: ""
4444
groups: "nodeport"
4545
kafka: "4.1.0"
@@ -90,7 +90,7 @@ scenarios:
9090
fixture: .github/tests/events/parse-comment/issue_comment_6.json
9191
expectations:
9292
pipelines: ""
93-
profiles: "operators,operands,brokers-and-security,azp_kraft_upgrade,azp_kafka_upgrade"
93+
profiles: ""
9494
tests: ""
9595
groups: ""
9696
kafka: "latest"
@@ -107,7 +107,7 @@ scenarios:
107107
fixture: .github/tests/events/parse-comment/issue_comment_7.json
108108
expectations:
109109
pipelines: ""
110-
profiles: "operators,operands,brokers-and-security,azp_kraft_upgrade,azp_kafka_upgrade"
110+
profiles: ""
111111
tests: ""
112112
groups: ""
113113
kafka: "latest"

‎.github/tests/scenarios/validate-matrix.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
scenarios:
2+
- id: valid-multiple-jobs
3+
description: "Validate matrix with multiple valid jobs"
4+
event: push
5+
matrix_file: .github/tests/inputs/validate-matrix/valid_multiple.json
6+
expectations:
7+
isValid: "true"
8+
message_contains: "✅ **System test validation passed**"
9+
10+
- id: valid-single-job
11+
description: "Validate matrix with single valid job"
12+
event: push
13+
matrix_file: .github/tests/inputs/validate-matrix/valid_single.json
14+
expectations:
15+
isValid: "true"
16+
message_contains: "✅ **System test validation passed**"
17+
18+
- id: empty-matrix
19+
description: "Validate empty matrix (should fail validation)"
20+
event: push
21+
matrix_file: .github/tests/inputs/validate-matrix/empty_matrix.json
22+
expectations:
23+
isValid: "false"
24+
message_contains: "❌ **System test validation failed**"
25+
26+
- id: null-matrix
27+
description: "Validate null matrix input"
28+
event: push
29+
matrix_file: .github/tests/inputs/validate-matrix/null_matrix.json
30+
expectations:
31+
isValid: "false"
32+
message_contains: "❌ **System test validation failed**"
33+
34+
- id: valid-with-custom-jobs
35+
description: "Validate matrix with custom job configurations"
36+
event: push
37+
matrix_file: .github/tests/inputs/validate-matrix/valid_custom.json
38+
expectations:
39+
isValid: "true"
40+
message_contains: "✅ **System test validation passed**"

‎.github/tests/workflows/validate-matrix-template.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Validate-matrix template
2+
3+
on:
4+
push:
5+
6+
jobs:
7+
test:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: actions/checkout@v5
11+
12+
# Test the validate-matrix action
13+
- id: validate-matrix
14+
uses: ./.github/actions/systemtests/validate-matrix
15+
with:
16+
matrix: ${{ env.TEST_MATRIX }}
17+
18+
# Validate the action outputs
19+
- name: Validate action outputs
20+
shell: bash
21+
env:
22+
EXPECT_ISVALID: ${{ env.EXPECT_ISVALID }}
23+
EXPECT_MESSAGE_CONTAINS: ${{ env.EXPECT_MESSAGE_CONTAINS }}
24+
EXPECT_ERROR_CONTAINS: ${{ env.EXPECT_ERROR_CONTAINS }}
25+
EXPECT_JOB_COUNT: ${{ env.EXPECT_JOB_COUNT }}
26+
run: |
27+
set +e # Don't exit on first error
28+
VALIDATION_ERRORS=0
29+
30+
IS_VALID='${{ steps.validate-matrix.outputs.isValid }}'
31+
MESSAGE='${{ steps.validate-matrix.outputs.message }}'
32+
33+
echo "::group::Action Outputs"
34+
echo "isValid: $IS_VALID"
35+
echo "message: $MESSAGE"
36+
echo "::endgroup::"
37+
38+
# Test expected values if provided
39+
if [[ -n "$EXPECT_ISVALID" ]]; then
40+
if [[ "$IS_VALID" == "$EXPECT_ISVALID" ]]; then
41+
echo "✅ isValid matches expectation: $EXPECT_ISVALID"
42+
else
43+
echo "❌ isValid mismatch. Expected: $EXPECT_ISVALID, Got: $IS_VALID"
44+
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
45+
fi
46+
fi
47+
48+
if [[ -n "$EXPECT_MESSAGE_CONTAINS" ]]; then
49+
if [[ "$MESSAGE" == *"$EXPECT_MESSAGE_CONTAINS"* ]]; then
50+
echo "✅ Message contains expected text: $EXPECT_MESSAGE_CONTAINS"
51+
else
52+
echo "❌ Message does not contain expected text: $EXPECT_MESSAGE_CONTAINS"
53+
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
54+
fi
55+
fi
56+
57+
# Summary
58+
if [[ $VALIDATION_ERRORS -eq 0 ]]; then
59+
echo "✅ All validations passed!"
60+
exit 0
61+
else
62+
echo "❌ Validation failed with $VALIDATION_ERRORS error(s)"
63+
exit 1
64+
fi

0 commit comments

Comments
 (0)