[ci] CUDA fallback test: run end-to-end with --lowmem + 30 min budget #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| cpp-build: | |
| name: build (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-22.04 | |
| binary: build/siamize | |
| deps_check: | | |
| echo "--- ldd ---" | |
| ldd build/siamize | |
| echo "--- direct (NEEDED) deps ---" | |
| readelf -d build/siamize | grep NEEDED | |
| - os: macos-latest | |
| binary: build/siamize | |
| deps_check: | | |
| echo "--- otool -L (linked dylibs) ---" | |
| otool -L build/siamize | |
| - os: windows-latest | |
| # MSVC multi-config generator drops the binary under Release/. | |
| binary: build/Release/siamize.exe | |
| deps_check: | | |
| ls -la build/Release/ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install OpenMP (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install libomp | |
| echo "OpenMP_ROOT=$(brew --prefix libomp)" >> "$GITHUB_ENV" | |
| - name: Fetch C++ deps | |
| run: scripts/fetch_deps.sh | |
| - name: Configure | |
| # -DSIAMIZE_STATIC_LINK=ON is the default but we pin it for clarity: | |
| # statically links libstdc++/libgcc/libgomp (Linux), libomp (macOS), | |
| # /MT static CRT (Windows). Only libonnxruntime remains dynamic. | |
| run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DSIAMIZE_STATIC_LINK=ON | |
| - name: Build | |
| run: cmake --build build --config Release --parallel | |
| - name: Inspect runtime deps | |
| run: ${{ matrix.deps_check }} | |
| - name: Smoke-run --help | |
| run: ${{ matrix.binary }} --help | |
| - name: Package | |
| shell: bash | |
| run: | | |
| BASE=$(cat VERSION 2>/dev/null || echo "0.0.0") | |
| # Tag push -> use tag verbatim (strip leading 'v' for the bundle). | |
| # Branch push -> base version + short SHA so successive main builds | |
| # are uniquely-named in the artifact list. | |
| if [ "${GITHUB_REF_TYPE}" = "tag" ]; then | |
| VER="${GITHUB_REF_NAME#v}" | |
| else | |
| VER="${BASE}-${GITHUB_SHA:0:7}" | |
| fi | |
| case "${{ runner.os }}" in | |
| Linux) PLAT=linux-x86_64 ;; | |
| macOS) PLAT=macos ;; | |
| Windows) PLAT=windows-x64 ;; | |
| esac | |
| BUNDLE="siamize-${VER}-${PLAT}-cpu" | |
| scripts/package.sh cpu "$BUNDLE" | |
| echo "BUNDLE=$BUNDLE" >> "$GITHUB_ENV" | |
| - name: Upload bundle | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.BUNDLE }} | |
| path: ${{ env.BUNDLE }}.zip | |
| if-no-files-found: error | |
| retention-days: 14 | |
| cpp-build-cuda: | |
| # Builds siamize with the CUDA Execution Provider enabled, then runs a | |
| # single-fold inference on the bundled sample. GitHub-hosted runners | |
| # have no NVIDIA GPU, so we expect siamize to register CUDA at startup, | |
| # fail to dlopen the CUDA runtime libraries, and silently fall back to | |
| # the CPU EP. This validates the build path AND the runtime fallback. | |
| # | |
| # The fp16 ONNX weight file (~270 MB raw, ~91 MB gzipped) is too | |
| # big for git, so CI grabs it from a public URL configured in the | |
| # repository's GitHub Actions variables: Settings -> Secrets and | |
| # variables -> Actions -> Variables -> SIAMIZE_WEIGHTS_URL is the | |
| # FULL URL of the fold_0 weight (e.g. | |
| # https://neurojson.org/io/stat.cgi?action=get&db=siam_v03&doc=dynshape&size=95360591&file=fold_0_fp16.onnx.gz | |
| # ). The URL may end in .gz (preferred, decompressed with gunzip), | |
| # .onnx (used raw), or .7z (legacy -- extracted with 7z if | |
| # available). The job is skipped if the variable is not configured. | |
| name: build+run cuda (${{ matrix.os }}) | |
| if: vars.SIAMIZE_WEIGHTS_URL != '' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-22.04 | |
| binary: build/siamize | |
| - os: windows-latest | |
| binary: build/Release/siamize.exe | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fetch C++ deps (GPU ORT prebuilt) | |
| env: | |
| ORT_BUILD: gpu | |
| run: scripts/fetch_deps.sh | |
| - name: Cache fold_0 weight | |
| id: cache-fold0 | |
| uses: actions/cache@v4 | |
| with: | |
| path: models/fold_0_fp16.onnx | |
| # Cache key bumped from v2 -> v3 to invalidate a stale | |
| # pre-dynamic_axes export. The previous v2 key kept serving | |
| # a fixed-shape (256,256,192) ONNX that rejected any other | |
| # patch size (e.g. --lowmem's 192x192x128). tools/onnx_export | |
| # now emits dynamic spatial axes (D, H, W) -- bump to v3 to | |
| # force a re-fetch from $SIAMIZE_WEIGHTS_URL. | |
| key: siam-fold0-fp16-v3 | |
| - name: Fetch + extract fold_0 weight (cache miss) | |
| if: steps.cache-fold0.outputs.cache-hit != 'true' | |
| env: | |
| URL: ${{ vars.SIAMIZE_WEIGHTS_URL }} | |
| run: | | |
| mkdir -p models | |
| echo "Downloading $URL" | |
| # Dispatch by the actual URL extension so the repo variable can | |
| # point at .gz (preferred), .onnx (uncompressed), or .7z (legacy) | |
| # without us guessing. | |
| case "$URL" in | |
| *.gz) | |
| curl -fsSL "$URL" -o models/fold_0_fp16.onnx.gz | |
| gunzip -f models/fold_0_fp16.onnx.gz | |
| ;; | |
| *.7z) | |
| if ! command -v 7z >/dev/null; then | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends p7zip-full | |
| fi | |
| curl -fsSL "$URL" -o models/fold_0_fp16.onnx.7z | |
| 7z e -y -omodels models/fold_0_fp16.onnx.7z >/dev/null | |
| rm -f models/fold_0_fp16.onnx.7z | |
| ;; | |
| *.onnx) | |
| curl -fsSL "$URL" -o models/fold_0_fp16.onnx | |
| ;; | |
| *) | |
| echo "unsupported extension for SIAMIZE_WEIGHTS_URL: $URL" | |
| echo "expected one of: .gz, .onnx, .7z" | |
| exit 1 | |
| ;; | |
| esac | |
| ls -la models/ | |
| - name: Configure (CUDA EP) | |
| run: | | |
| cmake -S . -B build \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DSIAMIZE_STATIC_LINK=ON \ | |
| -DSIAMIZE_GPU=cuda | |
| - name: Build | |
| run: cmake --build build --config Release --parallel | |
| - name: Run single-fold inference (expect CPU fallback on GPU-less runner) | |
| # GitHub-hosted runners have no NVIDIA GPU, so siamize will | |
| # detect that, log "[cuda] unavailable (...); using CPU", and | |
| # silently fall back to the CPU EP. We want to run the full | |
| # inference end-to-end as a real smoke test, not just verify | |
| # the [cuda] log line. | |
| # | |
| # Memory and time budget: | |
| # * Runner: 4 vCPU, 16 GB RAM | |
| # * Default 256x256x192 patch peaks at ~12 GB RSS, right at | |
| # the runner's cgroup ceiling -- earlier attempts hit | |
| # SIGTERM mid-inference. | |
| # * --lowmem (with the v3-cached dynamic-shape ONNX) shrinks | |
| # the patch to 192x192x128 and applies --no-arena + -t 8 | |
| # + cudnn-workspace 0 + gpu-mem-limit 6G -> peak RSS ~6 GB | |
| # and total wall time roughly the same as the default | |
| # (more tiles, but each tile is faster). | |
| timeout-minutes: 30 | |
| run: | | |
| ${{ matrix.binary }} \ | |
| --lowmem \ | |
| -i tests/sub-01_T1w.nii.gz \ | |
| -o pred_ci.nii.gz \ | |
| -M models/fold_0_fp16.onnx \ | |
| -c auto 2>&1 | tee siamize.log | |
| - name: Verify CPU fallback happened | |
| run: | | |
| if grep -Eq "\[cuda\][[:space:]]+unavailable" siamize.log; then | |
| echo "OK: CUDA EP correctly fell back to CPU." | |
| else | |
| echo "FAIL: expected '[cuda] unavailable' fallback message; got:" | |
| cat siamize.log | |
| exit 1 | |
| fi | |
| - name: Verify output produced | |
| run: | | |
| test -s pred_ci.nii.gz || { echo "FAIL: empty/missing output"; exit 1; } | |
| ls -lh pred_ci.nii.gz | |
| - name: Package | |
| shell: bash | |
| run: | | |
| BASE=$(cat VERSION 2>/dev/null || echo "0.0.0") | |
| if [ "${GITHUB_REF_TYPE}" = "tag" ]; then | |
| VER="${GITHUB_REF_NAME#v}" | |
| else | |
| VER="${BASE}-${GITHUB_SHA:0:7}" | |
| fi | |
| case "${{ runner.os }}" in | |
| Linux) PLAT=linux-x86_64 ;; | |
| Windows) PLAT=windows-x64 ;; | |
| esac | |
| BUNDLE="siamize-${VER}-${PLAT}-cuda" | |
| scripts/package.sh cuda "$BUNDLE" | |
| echo "BUNDLE=$BUNDLE" >> "$GITHUB_ENV" | |
| - name: Upload bundle | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.BUNDLE }} | |
| path: ${{ env.BUNDLE }}.zip | |
| if-no-files-found: error | |
| retention-days: 14 | |
| mex-build: | |
| # Build siamize's MATLAB/Octave MEX wrapper and verify it loads. Heavy | |
| # full-inference test is left to cpp-build-cuda; this job's job is just | |
| # to catch MEX build regressions on each supported host. | |
| name: mex (${{ matrix.label }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - label: linux-octave | |
| os: ubuntu-22.04 | |
| kind: octave | |
| cmake_flag: -DSIAMIZE_BUILD_OCTAVE_MEX=ON | |
| mex_file: build/siamex.mex | |
| - label: linux-matlab | |
| os: ubuntu-22.04 | |
| kind: matlab | |
| cmake_flag: -DSIAMIZE_BUILD_MATLAB_MEX=ON | |
| mex_file: build/siamex.mexa64 | |
| - label: windows-matlab | |
| os: windows-latest | |
| kind: matlab | |
| cmake_flag: -DSIAMIZE_BUILD_MATLAB_MEX=ON | |
| mex_file: build/Release/siamex.mexw64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Octave (Linux) | |
| if: matrix.kind == 'octave' && runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends octave liboctave-dev | |
| - name: Set up MATLAB | |
| if: matrix.kind == 'matlab' | |
| uses: matlab-actions/setup-matlab@v2 | |
| with: | |
| release: R2024a | |
| - name: Install GCC 10 (Linux MATLAB) | |
| # MATLAB R2024a on Linux bundles a libstdc++.so.6 that supports | |
| # GLIBCXX <=3.4.28 (GCC 9 / 10). The ubuntu-22.04 runner ships | |
| # GCC 11 as the default, whose libstdc++ headers reference | |
| # GLIBCXX_3.4.29 symbols and produce a MEX MATLAB can't load. | |
| # GCC 10 is MathWorks' officially supported MEX compiler for | |
| # R2024a on Linux. | |
| if: matrix.kind == 'matlab' && runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends g++-10 gcc-10 | |
| - name: Fetch C++ deps (CPU ORT) | |
| run: scripts/fetch_deps.sh | |
| - name: Configure | |
| env: | |
| # Pin to GCC 10 for the MATLAB-on-Linux leg so the MEX only | |
| # references GLIBCXX symbols MATLAB's bundled libstdc++ has. | |
| CC: ${{ (matrix.kind == 'matlab' && runner.os == 'Linux') && 'gcc-10' || '' }} | |
| CXX: ${{ (matrix.kind == 'matlab' && runner.os == 'Linux') && 'g++-10' || '' }} | |
| run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flag }} | |
| - name: Build | |
| run: cmake --build build --config Release --parallel | |
| - name: Verify MEX exists | |
| run: | | |
| ls -la "${{ matrix.mex_file }}" | |
| test -s "${{ matrix.mex_file }}" | |
| - name: Smoke-load MEX (Octave) | |
| if: matrix.kind == 'octave' && runner.os == 'Linux' | |
| run: | | |
| # Call siamex (the low-level MEX) with zero args to trigger the | |
| # C++ "Usage:" error. Proves the .mex loaded + reached mexFunction. | |
| octave-cli --no-gui --eval "addpath('build'); \ | |
| try; siamex(); catch err; \ | |
| if isempty(strfind(err.message, 'Usage:')); error('unexpected: %s', err.message); end; \ | |
| fprintf('MEX loaded and reached argument check\\n'); \ | |
| end" 2>&1 | tee mex.log | |
| grep -q "MEX loaded" mex.log | |
| - name: Run siamize.m unit tests (Octave) | |
| if: matrix.kind == 'octave' && runner.os == 'Linux' | |
| run: | | |
| # Pure-MATLAB dispatcher tests. Stubs siamex internally so it | |
| # doesn't depend on ORT or weight files; needs the jsonlab | |
| # submodule (pulled via submodules: recursive above). | |
| octave-cli --no-gui --eval "cd matlab/tests; run_tests('--exit')" | |
| - name: Smoke-load MEX (MATLAB) | |
| if: matrix.kind == 'matlab' | |
| uses: matlab-actions/run-command@v2 | |
| with: | |
| command: | | |
| addpath(fullfile(pwd, 'build')); | |
| % cd into build/Release on Windows so the MEX picks up neighboring | |
| % onnxruntime.dll via the EXE search rule. | |
| if ispc; cd(fullfile('build', 'Release')); end | |
| try | |
| siamex(); | |
| error('siamex() with no args should have thrown'); | |
| catch err | |
| if isempty(strfind(err.message, 'Usage:')) | |
| error('unexpected: %s', err.message); | |
| end | |
| fprintf('MEX loaded and reached argument check\n'); | |
| end | |
| - name: Run siamize.m unit tests (MATLAB) | |
| if: matrix.kind == 'matlab' | |
| uses: matlab-actions/run-command@v2 | |
| with: | |
| command: | | |
| cd(fullfile('matlab', 'tests')); | |
| [~, nfail] = run_tests(); | |
| assert(nfail == 0, 'siamize.m unit tests failed (%d)', nfail); | |
| - name: Package | |
| shell: bash | |
| run: | | |
| BASE=$(cat VERSION 2>/dev/null || echo "0.0.0") | |
| if [ "${GITHUB_REF_TYPE}" = "tag" ]; then | |
| VER="${GITHUB_REF_NAME#v}" | |
| else | |
| VER="${BASE}-${GITHUB_SHA:0:7}" | |
| fi | |
| BUNDLE="siamex-${VER}-${{ matrix.label }}" | |
| scripts/package.sh mex "$BUNDLE" | |
| echo "BUNDLE=$BUNDLE" >> "$GITHUB_ENV" | |
| - name: Upload bundle | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.BUNDLE }} | |
| path: ${{ env.BUNDLE }}.zip | |
| if-no-files-found: error | |
| retention-days: 14 | |
| python-smoke: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Python deps | |
| run: | | |
| pip install --upgrade pip | |
| pip install numpy nibabel scipy scikit-image torch \ | |
| dynamic_network_architectures onnxruntime | |
| - name: Import smoke test | |
| run: | | |
| python -c "import sys; sys.path.insert(0, 'py'); import siam_ref; print(siam_ref.build_network.__name__)" | |
| python tools/onnx_export/compare.py --help || true |