pr.yaml

name: Test PR

on:
  pull_request:
    paths-ignore: ["**/docker.yaml", "docs"]
  push:
    branches:
      - ci/*
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

env:
  NUM_WORKERS: auto
  USE_POOCH: "True"
  ECHOPYPE_DATA_VERSION: v0.11.1a2
  ECHOPYPE_DATA_BASEURL: https://github.com/OSOceanAcoustics/echopype/releases/download/{version}/
  XDG_CACHE_HOME: ${{ github.workspace }}/.cache

jobs:
  unit-tests:
    name: unit--${{ matrix.python-version }}--${{ matrix.runs-on }}
    runs-on: ${{ matrix.runs-on }}
    if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
        runs-on: [ubuntu-latest]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6.2.0
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Set environment variables
        run: echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV

      - name: Remove docker-compose python
        run: sed -i "/docker-compose/d" requirements-dev.txt

      - name: Install dev tools
        run: python -m pip install -r requirements-dev.txt

      - name: Install echopype
        run: python -m pip install -e ".[plot]"

      - name: Install pooch (needed by conftest)
        run: python -m pip install pooch

      - name: Start memory monitor
        shell: bash
        run: |
          mkdir -p ci_monitor
          nohup bash .ci_helpers/memory_usage/monitor_memory_linux.sh >/dev/null 2>&1 &
          echo $! > ci_monitor/monitor.pid

      - name: Run unit tests
        run: |
          pytest -m unit -vvv -rx --numprocesses=auto \
            --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings

      - name: Stop memory monitor
        if: always()
        shell: bash
        run: |
          if [ -f ci_monitor/monitor.pid ]; then
            kill "$(cat ci_monitor/monitor.pid)" || true
          fi

      - name: Upload memory CSV
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: memory-unit-${{ matrix.python-version }}-${{ matrix.runs-on }}
          path: ci_monitor/memory_usage.csv

      - name: Upload code coverage to Codecov
        uses: codecov/codecov-action@v6
        with:
          files: ./coverage.xml
          flags: unit
          env_vars: RUNNER_OS,PYTHON_VERSION
          name: codecov-unit
          fail_ci_if_error: false

  integration-tests:
    name: integration--${{ matrix.python-version }}--${{ matrix.runs-on }}
    runs-on: ${{ matrix.runs-on }}
    if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
    continue-on-error: ${{ matrix.experimental }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
        runs-on: [ubuntu-latest]
        experimental: [false]

    services:
      # TODO: figure out how to update tag when there's a new one
      minio:
        image: cormorack/minioci:latest
        ports:
          - 9000:9000
      httpserver:
        image: httpd:2.4
        ports:
          - 8080:80

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          fetch-depth: 0 # Fetch all history for all branches and tags.

      - name: Free disk space
        if: runner.os == 'Linux'
        run: |
          set -eux
          echo "Before cleanup:"
          df -h
          for p in /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL; do
            if [ -d "$p" ]; then
              echo "Removing $p (size: $(du -sh "$p" | cut -f1))"
              sudo rm -rf "$p"
            else
              echo "Not present: $p"
            fi
          done
          sudo apt-get clean
          sudo rm -rf /var/lib/apt/lists/*
          echo "After cleanup:"
          df -h

      - name: Set up Python
        uses: actions/setup-python@v6.2.0
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Set environment variables
        run: echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV

      - name: Remove docker-compose python
        run: sed -i "/docker-compose/d" requirements-dev.txt

      - name: Install dev tools
        run: python -m pip install -r requirements-dev.txt

      - name: Cache echopype test data
        uses: actions/cache@v5
        with:
          path: ${{ env.XDG_CACHE_HOME }}/echopype
          key: ep-data-${{ env.ECHOPYPE_DATA_VERSION }}

      - name: Ensure pooch (for GitHub assets)
        run: python -m pip install "pooch>=1.8"

      - name: Install error reporter
        if: ${{ matrix.python-version == '3.13' }}
        run: python -m pip install pytest-github-actions-annotate-failures

      - name: Install echopype
        run: python -m pip install -e ".[plot]"

      - name: Print installed packages
        run: python -m pip list

      - name: Finding changed files
        id: files
        uses: tj-actions/changed-files@v47

      - name: Print Changed files
        run: echo "${{ steps.files.outputs.all_changed_files }}"

      - name: Disk usage (before test data)
        if: runner.os == 'Linux'
        run: |
          set -eux
          df -h
          du -h -d 3 ${{ env.XDG_CACHE_HOME }} || true

      - name: Pre-fetch Pooch test data
        run: |
          pytest --collect-only -q

      - name: Copy test data to MinIO and HTTP server
        run: |
          python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}

          # Check data endpoint
          curl http://localhost:8080/data/

      - name: Disk usage (after services deploy)
        if: runner.os == 'Linux'
        run: |
          set -eux
          df -h
          du -h -d 3 ${{ env.XDG_CACHE_HOME }} || true
          docker system df || true

      - name: Start memory monitor
        shell: bash
        run: |
          mkdir -p ci_monitor
          nohup bash .ci_helpers/memory_usage/monitor_memory_linux.sh >/dev/null 2>&1 &
          echo $! > ci_monitor/monitor.pid

      - name: Running integration tests
        run: |
          pytest -m integration -vvv -rx --numprocesses=${{ env.NUM_WORKERS }} --max-worker-restart=3 \
          --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings

      - name: Stop memory monitor
        if: always()
        shell: bash
        run: |
          if [ -f ci_monitor/monitor.pid ]; then
            kill "$(cat ci_monitor/monitor.pid)" || true
          fi

      - name: Upload memory CSV
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: memory-integration-${{ matrix.python-version }}-${{ matrix.runs-on }}
          path: ci_monitor/memory_usage.csv

      - name: Upload code coverage to Codecov
        uses: codecov/codecov-action@v6
        with:
          files: ./coverage.xml
          flags: integration
          env_vars: RUNNER_OS,PYTHON_VERSION
          name: codecov-integration
          fail_ci_if_error: false

  test-windows:
    name: ${{ matrix.python-version }}--windows
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13"]

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v6.2.0
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install deps (dev + pooch + s3)
        run: |
          python -m pip install -r requirements-dev.txt
          python -m pip install "pooch>=1.8" fsspec s3fs
          python -m pip install pytest pytest-cov pytest-xdist pytest-mock

      - name: Install error reporter
        if: ${{ matrix.python-version == '3.13' }}
        run: python -m pip install pytest-github-actions-annotate-failures

      - name: Install echopype
        run: python -m pip install -e ".[plot]"

      - name: Pre-fetch all Pooch assets
        run: python -m pytest --collect-only -q

      - name: Start local services
        run: python .ci_helpers/setup-services-windows.py start

      - name: Start memory monitor
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Force -Path ci_monitor | Out-Null
          $p = Start-Process pwsh -ArgumentList "-File", ".ci_helpers/memory_usage/monitor_memory_windows.ps1" -PassThru -WindowStyle Hidden
          $p.Id | Out-File ci_monitor/monitor.pid -Encoding ascii

      - name: Running all tests
        shell: pwsh
        env:
          PY_COLORS: "1"
        run: >
          python -m pytest -vvv -rx
          --numprocesses $env:NUM_WORKERS --max-worker-restart 3
          --cov echopype --cov-report xml
          --log-cli-level WARNING --disable-warnings

      - name: Stop memory monitor
        if: always()
        shell: pwsh
        run: |
          if (Test-Path ci_monitor/monitor.pid) {
            $monitorPid = Get-Content ci_monitor/monitor.pid
            Stop-Process -Id $monitorPid -Force -ErrorAction SilentlyContinue
          }

      - name: Upload memory CSV
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: memory-windows-${{ matrix.python-version }}
          path: ci_monitor/memory_usage.csv

      - name: Upload code coverage
        uses: codecov/codecov-action@v6
        with:
          files: ./coverage.xml
          flags: unittests
          env_vars: RUNNER_OS
          name: codecov-windows
          fail_ci_if_error: false

      - name: Teardown services
        if: always()
        run: python .ci_helpers/setup-services-windows.py stop

  memory-summary:
    name: memory-summary
    runs-on: ubuntu-latest
    needs: [unit-tests, integration-tests, test-windows]
    if: always()

    steps:
      - uses: actions/checkout@v6

      - name: Download all memory artifacts
        uses: actions/download-artifact@v8
        with:
          path: memory_artifacts

      - name: Install matplotlib
        run: python -m pip install matplotlib

      - name: Build memory summary
        run: python .ci_helpers/memory_usage/build_memory_summary.py

      - name: Upload combined memory plot
        uses: actions/upload-artifact@v7
        with:
          name: memory-summary
          path: memory_all_jobs.png