system-tests.yml

name: System Tests

on:
  # Trigger for manual execution from GitHub UI
  workflow_dispatch:
    inputs:
      releaseVersion:
        description: "Release version to test (i.e. 'latest', '0.46.0')"
        required: false
        default: "latest"
      kafkaVersion:
        description: "Kafka version. e.g. '4.0.0' or 'latest'"
        required: false
        default: "latest"
      pipeline:
        description: "Comma-separated or single pipeline name(s)"
        required: false
        default: "regression,upgrade,regression-fg,acceptance-helm"
        type: string
  # Automated trigger for issue comment
  issue_comment:
    types: [created]
  # Testing trigger for PRs - tests workflow changes
  pull_request:
    types: [opened, synchronize]
    paths:
      - '.github/workflows/run-system-tests.yml'
      - '.github/workflows/system-tests.yml'

permissions:
  contents: read

jobs:
  # Parse and build parameters from comment or from workflow params
  parse-params:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: write
      statuses: write
    outputs:
      ref: ${{ steps.determine_ref.outputs.ref }}
      sha: ${{ steps.determine_ref.outputs.sha }}
      shouldRun: ${{ steps.should_run.outputs.shouldRun }}
    steps:
      - name: Should Run
        id: should_run
        uses: strimzi/github-actions/.github/actions/utils/should-run@v1
      - name: Determine checkout ref
        id: determine_ref
        if: steps.should_run.outputs.shouldRun == 'true'
        uses: strimzi/github-actions/.github/actions/utils/determine-ref@v1

  check-rights:
    needs:
      - parse-params
    if: ${{ needs.parse-params.outputs.shouldRun == 'true' && github.event_name != 'pull_request' }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: write
      statuses: write
    steps:
      - uses: strimzi/github-actions/.github/actions/utils/check-permissions@v1

  check-build:
    needs:
      - check-rights
      - parse-params
    if: |-
      ${{ always() &&
        (needs.check-rights.result == 'success' || github.event_name == 'pull_request') &&
        needs.parse-params.outputs.shouldRun == 'true'
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 90
    permissions:
      contents: read
      pull-requests: write
    outputs:
      buildRunId: ${{ steps.verify.outputs.buildRunId }}
      buildMetadataSha: ${{ steps.verify.outputs.buildMetadataSha }}
      commentId: ${{ steps.initial-comment.outputs.commentId }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}
      - name: Add initial comment
        id: initial-comment
        if: ${{ github.event_name == 'issue_comment' }}
        uses: ./.github/actions/utils/add-comment
        with:
          commentMessage: ':hourglass_flowing_sand: System test verification started: [link](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

          Waiting for build to finish...'
      - name: Get merge commit SHA
        id: get-merge-sha
        run: |
          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
      - name: Compare merge-commit sha
        id: verify
        uses: ./.github/actions/utils/compare-merge-commit
        with:
          waitForBuild: "true"
          maxWaitMinutes: "80"
          mergeCommitSha: ${{ steps.get-merge-sha.outputs.sha }}
          commitSha: ${{ needs.parse-params.outputs.sha }}

  generate-matrix:
    needs:
      - parse-params
      - check-rights
      - check-build
    if: |-
      ${{
        always() &&
        needs.parse-params.outputs.shouldRun == 'true' &&
        (needs.check-rights.result == 'success' || github.event_name == 'pull_request') &&
        needs.check-build.result == 'success'
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: write
      statuses: write
    outputs:
      pipelineList: ${{ steps.parse.outputs.pipelineList }}
      profileList: ${{ steps.parse.outputs.profileList }}
      agent: ${{ steps.parse.outputs.agent }}
      architecture: ${{ steps.parse.outputs.architecture }}
      strimzi_feature_gates: ${{ steps.parse.outputs.strimzi_feature_gates }}
      strimzi_rbac_scope: ${{ steps.parse.outputs.strimzi_rbac_scope }}
      cluster_operator_install_type: ${{ steps.parse.outputs.cluster_operator_install_type }}
      groups: ${{ steps.parse.outputs.groups }}
      tests: ${{ steps.parse.outputs.tests }}
      kafkaVersion: ${{ steps.parse.outputs.kafkaVersion }}
      releaseVersion: ${{ steps.parse.outputs.releaseVersion }}
      parallel: ${{ steps.parse.outputs.parallel }}
      kubeVersion: ${{ steps.parse.outputs.kubeVersion }}
      kindVersion: ${{ steps.parse.outputs.kindVersion }}
      isValid: ${{ steps.validate.outputs.isValid }}
      message: ${{ steps.validate.outputs.message }}
      matrix: ${{ steps.gen.outputs.matrix }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}
      - name: Parse Comment
        id: parse
        uses: ./.github/actions/systemtests/parse-comment
        with:
          # Parameters from manual trigger via UI
          releaseVersion: ${{ github.event.inputs.releaseVersion }}
          kafkaVersion: ${{ github.event.inputs.kafkaVersion }}
          pipeline: ${{ github.event.inputs.pipeline }}
      - name: Generate Matrix for Validation
        id: gen
        uses: ./.github/actions/systemtests/generate-matrix
        with:
          pipelines: ${{ steps.parse.outputs.pipelineList }}
          profiles: ${{ steps.parse.outputs.profileList }}
          default_agent: ${{ steps.parse.outputs.agent }}
          default_arch: ${{ steps.parse.outputs.architecture }}
          default_strimzi_feature_gates: ${{ steps.parse.outputs.strimzi_feature_gates }}
          default_strimzi_rbac_scope: ${{ steps.parse.outputs.strimzi_rbac_scope }}
          default_cluster_operator_install_type: ${{ steps.parse.outputs.cluster_operator_install_type }}
          default_parallel: ${{ steps.parse.outputs.parallel }}
          default_tests: ${{ steps.parse.outputs.tests }}
          default_groups: ${{ steps.parse.outputs.groups }}
          runnerArch: "amd64"
      - name: Validate Matrix
        id: validate
        uses: ./.github/actions/systemtests/validate-matrix
        with:
          matrix: ${{ steps.gen.outputs.matrix }}
      - name: Add validation error comment
        if: ${{ github.event_name == 'issue_comment' && steps.validate.outputs.isValid == 'false' }}
        uses: ./.github/actions/utils/add-comment
        with:
          commentMessage: ${{ steps.validate.outputs.message }}
      - name: Add trigger comment
        if: ${{ github.event_name == 'issue_comment' && steps.validate.outputs.isValid == 'true' }}
        uses: ./.github/actions/utils/add-comment
        with:
          commentId: ${{ needs.check-build.outputs.commentId }}
          commentMessage: ':hourglass_flowing_sand: System test verification started: [link](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

          ${{ steps.validate.outputs.message }}'
      - name: Set check & commit status (pending)
        if: ${{ steps.validate.outputs.isValid == 'true' }}
        uses: ./.github/actions/utils/check-and-status
        with:
          checkState: pending
          checkName: "System tests verification"
          checkDescription: "Check for overall system test verification"

  build-artifacts:
    name: build-artifacts
    needs:
      - parse-params
      - check-rights
      - generate-matrix
      - check-build
    # Run only if properly triggered, permissions check passed, matrix is valid, and the run is not for release (already built images)
    if: |-
      ${{ 
        always() &&needs.parse-params.outputs.shouldRun == 'true' && 
        (needs.check-rights.result == 'success' || github.event_name == 'pull_request') && 
        needs.generate-matrix.outputs.isValid == 'true' && 
        needs.generate-matrix.outputs.releaseVersion == 'latest' &&
        needs.check-build.outputs.buildRunId == ''
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}

      - name: Setup Java and Maven
        uses: frawless/github-actions/.github/actions/dependencies/setup-java@v6.1
        with:
          javaVersion: "21"

      - name: Install yq
        uses: strimzi/github-actions/.github/actions/dependencies/install-yq@v1
        with:
          version: "v4.6.3"

      - name: Install Shellcheck
        uses: strimzi/github-actions/.github/actions/dependencies/install-shellcheck@v1
        with:
          version: "0.11.0"

      - name: Install Helm
        uses: strimzi/github-actions/.github/actions/dependencies/install-helm@v1
        with:
          helmVersion: "v3.16.0"

      - name: Build binaries using build-binaries action
        uses: strimzi/github-actions/.github/actions/build/build-binaries@v1
        with:
          mainJavaBuild: "true"
          artifactSuffix: "operators"
          clusterOperatorBuild: "true"
        env:
          MVN_ARGS: "-DskipTests -Dmaven.javadoc.skip=true"

  build-images:
    name: build-images
    needs:
      - parse-params
      - check-rights
      - generate-matrix
      - check-build
      - build-artifacts
    # Run only if properly triggered, permissions check passed, matrix is valid, and the run is not for release (already built images)
    if: |-
      ${{ 
        always() && needs.parse-params.outputs.shouldRun == 'true' && 
        (needs.check-rights.result == 'success' || github.event_name == 'pull_request')&& 
        needs.generate-matrix.outputs.isValid == 'true' && 
        needs.generate-matrix.outputs.releaseVersion == 'latest' && 
        needs.build-artifacts.result != 'failure' &&
        needs.check-build.outputs.buildRunId == ''
      }}
    strategy:
      matrix:
        architecture: [amd64, arm64]
    runs-on: oracle-vm-2cpu-8gb-arm64
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}

      - name: Install Docker
        uses: strimzi/github-actions/.github/actions/dependencies/install-docker@v1

      - name: Install yq
        uses: strimzi/github-actions/.github/actions/dependencies/install-yq@v1
        with:
          version: "v4.6.3"

      - name: Install Shellcheck
        uses: strimzi/github-actions/.github/actions/dependencies/install-shellcheck@v1
        with:
          version: "0.11.0"

      - uses: strimzi/github-actions/.github/actions/build/build-containers@v1
        with:
          architecture: ${{ matrix.architecture }}
          imagesDir: "./docker-images/container-archives"
          artifactSuffix: "operators"

  run-tests:
    name: run-tests
    needs:
      - parse-params
      - check-rights
      - check-build
      - generate-matrix
      - build-artifacts
      - build-images
    # Run if properly triggered, permissions check passed, matrix is valid, and build jobs succeeded or were skipped
    if: |-
      ${{
        always() &&
        needs.parse-params.outputs.shouldRun == 'true' &&
        (needs.check-rights.result == 'success' || github.event_name == 'pull_request') &&
        needs.generate-matrix.outputs.isValid == 'true' &&
        needs.check-build.result == 'success' &&
        (needs.build-images.result == 'success' || needs.build-images.result == 'skipped')
      }}
    # Job needs write rights for setting statuses on pull-requests
    permissions:
      contents: read
      pull-requests: write
      statuses: write
    uses: ./.github/workflows/run-system-tests.yml
    with:
      releaseVersion: ${{ needs.generate-matrix.outputs.releaseVersion }}
      kafkaVersion: ${{ needs.generate-matrix.outputs.kafkaVersion }}
      kubeVersion: ${{ needs.generate-matrix.outputs.kubeVersion }}
      kindVersion: ${{ needs.generate-matrix.outputs.kindVersion }}
      # Checkout ref params
      ref: ${{ needs.parse-params.outputs.ref }}
      sha: ${{ needs.parse-params.outputs.sha }}
      # Pass the validated matrix directly
      matrix: ${{ needs.generate-matrix.outputs.matrix }}
      # Build verification outputs
      buildRunId: ${{ needs.check-build.outputs.buildRunId }}

  report:
    needs:
      - parse-params
      - check-rights
      - generate-matrix
      - run-tests
    if: |-
      ${{
        always() &&
        needs.parse-params.outputs.shouldRun == 'true' &&
        needs.check-rights.result == 'success' &&
        github.event_name != 'pull_request' &&
        needs.generate-matrix.outputs.isValid == 'true'
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: write
      statuses: write
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}
      - name: Add comment
        uses: ./.github/actions/utils/add-comment
        with:
          commentMessage: |
            ${{
              (needs.run-tests.result == 'success' && ':tada: System test verification **passed**') ||
              (needs.run-tests.result == 'failure' && ':x: System test verification **failed**') ||
              ':warning: System test verification **skipped**'
            }}: [link](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
      - name: Set check & commit status
        uses: ./.github/actions/utils/check-and-status
        with:
          checkState: ${{ needs.run-tests.result == 'skipped' && 'neutral' || needs.run-tests.result }}
          checkName: "System tests verification"
          checkDescription: "Check for overall system test verification"

  perf-report:
    name: Performance Report
    needs:
      - parse-params
      - check-rights
      - generate-matrix
      - run-tests
    # Only run if performance profile/pipeline was executed
    if: |-
      ${{
        always() &&
        needs.parse-params.outputs.shouldRun == 'true' &&
        needs.check-rights.result == 'success' &&
        github.event_name != 'pull_request' &&
        needs.generate-matrix.outputs.isValid == 'true' &&
        (contains(needs.generate-matrix.outputs.profileList, 'performance') || contains(needs.generate-matrix.outputs.pipelineList, 'performance'))
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.parse-params.outputs.ref }}
      - uses: ./.github/actions/systemtests/run-perf-report