Skip to content
Test PR

Fix dev setup issues with pooch data fetch and cache handling #3897

Sign in to view logs

Fix dev setup issues with pooch data fetch and cache handling

Fix dev setup issues with pooch data fetch and cache handling #3897

Workflow file for this run

.github/workflows/pr.yaml at 2c35bc9
name: Test PR
on:
pull_request:
paths-ignore: ["**/docker.yaml", "docs"]
push:
branches:
- ci/*
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
env:
NUM_WORKERS: auto
USE_POOCH: "True"
ECHOPYPE_DATA_VERSION: v0.11.1a2
ECHOPYPE_DATA_BASEURL: https://github.com/OSOceanAcoustics/echopype/releases/download/{version}/
XDG_CACHE_HOME: ${{ github.workspace }}/.cache
jobs:
unit-tests:
name: unit--${{ matrix.python-version }}--${{ matrix.runs-on }}
runs-on: ${{ matrix.runs-on }}
if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
runs-on: [ubuntu-latest]
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6.2.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Upgrade pip
run: python -m pip install --upgrade pip
- name: Set environment variables
run: echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Remove docker-compose python
run: sed -i "/docker-compose/d" requirements-dev.txt
- name: Install dev tools
run: python -m pip install -r requirements-dev.txt
- name: Install echopype
run: python -m pip install -e ".[plot]"
- name: Install pooch (needed by conftest)
run: python -m pip install pooch
- name: Start memory monitor
shell: bash
run: |
mkdir -p ci_monitor
nohup bash .ci_helpers/memory_usage/monitor_memory_linux.sh >/dev/null 2>&1 &
echo $! > ci_monitor/monitor.pid
- name: Run unit tests
run: |
pytest -m unit -vvv -rx --numprocesses=auto \
--cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings
- name: Stop memory monitor
if: always()
shell: bash
run: |
if [ -f ci_monitor/monitor.pid ]; then
kill "$(cat ci_monitor/monitor.pid)" || true
fi
- name: Upload memory CSV
if: always()
uses: actions/upload-artifact@v7
with:
name: memory-unit-${{ matrix.python-version }}-${{ matrix.runs-on }}
path: ci_monitor/memory_usage.csv
- name: Upload code coverage to Codecov
uses: codecov/codecov-action@v6
with:
files: ./coverage.xml
flags: unit
env_vars: RUNNER_OS,PYTHON_VERSION
name: codecov-unit
fail_ci_if_error: false
integration-tests:
name: integration--${{ matrix.python-version }}--${{ matrix.runs-on }}
runs-on: ${{ matrix.runs-on }}
if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
runs-on: [ubuntu-latest]
experimental: [false]
services:
# TODO: figure out how to update tag when there's a new one
minio:
image: cormorack/minioci:latest
ports:
- 9000:9000
httpserver:
image: httpd:2.4
ports:
- 8080:80
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for all branches and tags.
- name: Free disk space
if: runner.os == 'Linux'
run: |
set -eux
echo "Before cleanup:"
df -h
for p in /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL; do
if [ -d "$p" ]; then
echo "Removing $p (size: $(du -sh "$p" | cut -f1))"
sudo rm -rf "$p"
else
echo "Not present: $p"
fi
done
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
echo "After cleanup:"
df -h
- name: Set up Python
uses: actions/setup-python@v6.2.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Upgrade pip
run: python -m pip install --upgrade pip
- name: Set environment variables
run: echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Remove docker-compose python
run: sed -i "/docker-compose/d" requirements-dev.txt
- name: Install dev tools
run: python -m pip install -r requirements-dev.txt
- name: Cache echopype test data
uses: actions/cache@v5
with:
path: ${{ env.XDG_CACHE_HOME }}/echopype
key: ep-data-${{ env.ECHOPYPE_DATA_VERSION }}
- name: Ensure pooch (for GitHub assets)
run: python -m pip install "pooch>=1.8"
- name: Install error reporter
if: ${{ matrix.python-version == '3.13' }}
run: python -m pip install pytest-github-actions-annotate-failures
- name: Install echopype
run: python -m pip install -e ".[plot]"
- name: Print installed packages
run: python -m pip list
- name: Finding changed files
id: files
uses: tj-actions/changed-files@v47
- name: Print Changed files
run: echo "${{ steps.files.outputs.all_changed_files }}"
- name: Disk usage (before test data)
if: runner.os == 'Linux'
run: |
set -eux
df -h
du -h -d 3 ${{ env.XDG_CACHE_HOME }} || true
- name: Pre-fetch Pooch test data
run: |
pytest --collect-only -q
- name: Copy test data to MinIO and HTTP server
run: |
python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}
# Check data endpoint
curl http://localhost:8080/data/
- name: Disk usage (after services deploy)
if: runner.os == 'Linux'
run: |
set -eux
df -h
du -h -d 3 ${{ env.XDG_CACHE_HOME }} || true
docker system df || true
- name: Start memory monitor
shell: bash
run: |
mkdir -p ci_monitor
nohup bash .ci_helpers/memory_usage/monitor_memory_linux.sh >/dev/null 2>&1 &
echo $! > ci_monitor/monitor.pid
- name: Running integration tests
run: |
pytest -m integration -vvv -rx --numprocesses=${{ env.NUM_WORKERS }} --max-worker-restart=3 \
--cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings
- name: Stop memory monitor
if: always()
shell: bash
run: |
if [ -f ci_monitor/monitor.pid ]; then
kill "$(cat ci_monitor/monitor.pid)" || true
fi
- name: Upload memory CSV
if: always()
uses: actions/upload-artifact@v7
with:
name: memory-integration-${{ matrix.python-version }}-${{ matrix.runs-on }}
path: ci_monitor/memory_usage.csv
- name: Upload code coverage to Codecov
uses: codecov/codecov-action@v6
with:
files: ./coverage.xml
flags: integration
env_vars: RUNNER_OS,PYTHON_VERSION
name: codecov-integration
fail_ci_if_error: false
test-windows:
name: ${{ matrix.python-version }}--windows
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-python@v6.2.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Upgrade pip
run: python -m pip install --upgrade pip
- name: Install deps (dev + pooch + s3)
run: |
python -m pip install -r requirements-dev.txt
python -m pip install "pooch>=1.8" fsspec s3fs
python -m pip install pytest pytest-cov pytest-xdist pytest-mock
- name: Install error reporter
if: ${{ matrix.python-version == '3.13' }}
run: python -m pip install pytest-github-actions-annotate-failures
- name: Install echopype
run: python -m pip install -e ".[plot]"
- name: Pre-fetch all Pooch assets
run: python -m pytest --collect-only -q
- name: Start local services
run: python .ci_helpers/setup-services-windows.py start
- name: Start memory monitor
shell: pwsh
run: |
New-Item -ItemType Directory -Force -Path ci_monitor | Out-Null
$p = Start-Process pwsh -ArgumentList "-File", ".ci_helpers/memory_usage/monitor_memory_windows.ps1" -PassThru -WindowStyle Hidden
$p.Id | Out-File ci_monitor/monitor.pid -Encoding ascii
- name: Running all tests
shell: pwsh
env:
PY_COLORS: "1"
run: >
python -m pytest -vvv -rx
--numprocesses $env:NUM_WORKERS --max-worker-restart 3
--cov echopype --cov-report xml
--log-cli-level WARNING --disable-warnings
- name: Stop memory monitor
if: always()
shell: pwsh
run: |
if (Test-Path ci_monitor/monitor.pid) {
$monitorPid = Get-Content ci_monitor/monitor.pid
Stop-Process -Id $monitorPid -Force -ErrorAction SilentlyContinue
}
- name: Upload memory CSV
if: always()
uses: actions/upload-artifact@v7
with:
name: memory-windows-${{ matrix.python-version }}
path: ci_monitor/memory_usage.csv
- name: Upload code coverage
uses: codecov/codecov-action@v6
with:
files: ./coverage.xml
flags: unittests
env_vars: RUNNER_OS
name: codecov-windows
fail_ci_if_error: false
- name: Teardown services
if: always()
run: python .ci_helpers/setup-services-windows.py stop
memory-summary:
name: memory-summary
runs-on: ubuntu-latest
needs: [unit-tests, integration-tests, test-windows]
if: always()
steps:
- uses: actions/checkout@v6
- name: Download all memory artifacts
uses: actions/download-artifact@v8
with:
path: memory_artifacts
- name: Install matplotlib
run: python -m pip install matplotlib
- name: Build memory summary
run: python .ci_helpers/memory_usage/build_memory_summary.py
- name: Upload combined memory plot
uses: actions/upload-artifact@v7
with:
name: memory-summary
path: memory_all_jobs.png