diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..08daf2d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,183 @@ +name: CI +on: + pull_request: + paths: + # We only want to run CI on spack versions that don't utilise symlinks (AKA pre-v1.1), + # in order to keep the CI simple. This means that the below pattern is essentially + # everthing in the v* directories, excluding the symlinked v0.* spack versions. + - 'v**' + - '!v0.**' + workflow_dispatch: + inputs: + spack-config-directories-to-test: + description: 'Space-separated list of version-based spack-config directories to test against' + required: true + type: string + default: v1.1 + spack-config-ref: + description: 'spack-config ref to use' + required: true + type: string + spack-ref: + description: 'spack ref to use' + required: true + type: string + run-self-hosted: + description: 'Use self-hosted runners for the test' + required: true + default: true + type: boolean +jobs: + setup-ci: + name: Setup CI + runs-on: ubuntu-latest + env: + # Note, this needs to be updated in line with refs in the CI job below + BUILD_CI_WORKFLOW_VERSION: v3 + # Which repository we get the manifest-to-test from + SPACK_MANIFEST_REPOSITORY: ACCESS-NRI/access-spack-packages + outputs: + spack-manifest-repository: ${{ env.SPACK_MANIFEST_REPOSITORY }} + # Matrices used in CI job + manifest-matrix: ${{ steps.set-manifest-matrix.outputs.matrix }} + config-matrix: ${{ steps.set-config-matrix-pr.outputs.all_changed_files || steps.set-config-matrix-dispatch.outputs.dirs }} + # Global defaults for CI job + spack-config-ref: ${{ inputs.spack-config-ref || github.event.pull_request.head.sha }} + spack-ref: ${{ inputs.spack-ref || steps.defaults.outputs.spack-ref }} + steps: + - name: Checkout spack-config + uses: actions/checkout@v6 + + - name: PR - Get dirs changed + id: set-config-matrix-pr + if: github.event_name == 'pull_request' + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + dir_names: true # we want a list of spack configuration directories (eg. v1.1, v1.2) + dir_names_max_depth: 1 # and don't care about subdirectories + dir_names_exclude_current_dir: true # or files changed in the root directory + matrix: true # and format it in a way understandable to GitHub actions matrices + # Below is an equivalent condition to on.pull_request.paths + files: | + v*/** + !v0.*/** + + - name: Dispatch - Get config directory to test + id: set-config-matrix-dispatch + if: github.event_name == 'workflow_dispatch' + run: | + echo "dirs=${{ inputs.spack-config-directories-to-test }}" >> $GITHUB_OUTPUT + + - name: Get ${{ env.SPACK_MANIFEST_REPOSITORY }} + uses: actions/checkout@v6 + with: + repository: ${{ env.SPACK_MANIFEST_REPOSITORY }} + + # This access-spack-packages matrix creation logic was adapted from + # https://github.com/ACCESS-NRI/access-spack-packages/blob/51064dc720f49c49e9680c368201e26081d4fa3f/.github/workflows/ci.yml#L58-L145 + # Since these manifests use the jinja template variable 'package', we need it here too. + # Future manifest repositories may not need to associate the spec with a jinja variable, in which case this can be removed. + - name: Get Manifests from ${{ env.SPACK_MANIFEST_REPOSITORY }} + id: access-spack-packages + env: + ACCESS_SPACK_PACKAGES_ROOT_DIR: spack_repo/access/nri/packages + run: | + pkgs=$(find ${{ env.ACCESS_SPACK_PACKAGES_ROOT_DIR }}/ -mindepth 1 -maxdepth 1 -type d -printf '%P ') + + echo "Packages found in access-spack-packages: $pkgs" + + echo "pkgs=$pkgs" >> $GITHUB_OUTPUT + + - name: Setup Manifest Matrix + id: set-manifest-matrix + # Get all packages that will be tested, then + # Get file paths to manifests for each package, and a template, then finally + # Convert into a JSON array for the matrix, one of the form: + # [ + # {"template_value": "mom5", "filepath": ".github/build-ci/manifests/mom5/intel.spack.yaml.j2"}, + # {"template_value": "mom5", "filepath": ".github/build-ci/manifests/mom5/gcc.spack.yaml.j2"}, + # {"template_value": "cice5", "filepath": ".github/build-ci/manifests/cice5/spack.yaml.j2"}, + # ... + # ] + run: | + pkgs="${{ steps.access-spack-packages.outputs.pkgs }}" + + for pkg in $pkgs; do + # We want each manifest to have an associated injection of {{ package }} to $pkg + # Replace underscores with hyphens for the template value, as spack converts directory underscores to package name hyphens + template_value="${pkg//_/-}" + + if [ -d ".github/build-ci/manifests/$pkg" ]; then # Look for specific manifests for the package + # We space-separate the paths as we later use them in a for loop + manifest_paths=$(find .github/build-ci/manifests/$pkg -iname '*.j2' -type f -printf '%p ') + fi + + if [[ -z "$manifest_paths" ]]; then # Use the default templates if no specific manifests exist for the package + # Similarly, space-separate the paths for a later for loop. Default manifests are always directly under .github/build-ci/manifests + manifest_paths=$(find .github/build-ci/manifests/ -maxdepth 1 -type f -iname '*.j2' -printf '%p ') + fi + + for manifest_path in $manifest_paths; do + json_entry=$(printf '{"template_value": "%s", "filepath": "%s"}' "$template_value" "$manifest_path") + json_entries+="${json_entry}," + done + unset manifest_paths + done + + echo "$json_entries" + + # Remove the trailing comma and wrap in square brackets + echo "matrix=[${json_entries%,}]" >> $GITHUB_OUTPUT + + # End of adapted package generation logic... + + - name: Checkout build-ci + uses: actions/checkout@v6 + with: + repository: access-nri/build-ci + ref: ${{ env.BUILD_CI_WORKFLOW_VERSION }} + + - name: Get default refs from build-ci workflow + id: defaults + # GitHub does not treat empty-string inputs as eligible for substitution with a default + # Since we are supporting both workflow_dispatch and pull_request triggers, the entrypoint inputs need to be filled in + # So we are getting the default values from the workflow file itself, and passing them back into the workflow + run: | + default_spack_ref=$(yq '.on.workflow_call.inputs."spack-ref".default' .github/workflows/ci.yml) + + echo "Default refs from ${{ env.BUILD_CI_WORKFLOW_VERSION }}: spack-ref=$default_spack_ref" + + echo "spack-ref=$default_spack_ref" >> $GITHUB_OUTPUT + + ci: + name: CI + needs: + - setup-ci + strategy: + fail-fast: false + max-parallel: 5 + matrix: + manifest: ${{ fromJson(needs.setup-ci.outputs.manifest-matrix) }} + config: ${{ fromJson(needs.setup-ci.outputs.config-matrix) }} + # Respectively of the form: + # [ + # {"template_value": "mom5", "filepath": ".github/build-ci/manifests/mom5/intel.spack.yaml.j2"}, + # {"template_value": "mom5", "filepath": ".github/build-ci/manifests/mom5/gcc.spack.yaml.j2"}, + # ... + # ] + # And: + # [v1.1, v1.3, ...] + uses: access-nri/build-ci/.github/workflows/ci.yml@v3 + with: + spack-manifest-repository: ${{ needs.setup-ci.outputs.spack-manifest-repository }} + spack-manifest-path: ${{ matrix.manifest.filepath }} + spack-manifest-data-path: .github/build-ci/data/standard_definitions.json + spack-manifest-data-pairs: |- + package ${{ matrix.manifest.template_value }} + spack-config-ref: ${{ needs.setup-ci.outputs.spack-config-ref }} + spack-ref: ${{ needs.setup-ci.outputs.spack-ref }} + container-image-version: :rocky-${{ matrix.config }} + # Workflow dispatch can switch depending on cluster/GitHub resourcing, otherwise run self-hosted + run-self-hosted: ${{ inputs.run-self-hosted || true }} + secrets: + spack-install-command-pat: ${{ secrets.SPACK_INSTALL_COMMAND_PAT }} diff --git a/v1.1/ci/downstream/config.yaml b/v1.1/ci/downstream/config.yaml index 896165f..de08a6d 100644 --- a/v1.1/ci/downstream/config.yaml +++ b/v1.1/ci/downstream/config.yaml @@ -1,3 +1,4 @@ +# Testing! config: install_tree: # Completely ignoring lower-precedence configuration options is supported with the :: notation for keys diff --git a/v1.1/toolchains.yaml b/v1.1/toolchains.yaml index 14425d2..68f725b 100644 --- a/v1.1/toolchains.yaml +++ b/v1.1/toolchains.yaml @@ -1,3 +1,4 @@ +# Testing out a change... toolchains: access_gcc: - spec: '%c=gcc'