Publish-only rollout endpoint #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ################################################################################ | |
| # This file is auto-generated from the .j2 file via generate_github_workflows.py. Do not edit manually. | |
| ################################################################################ | |
| name: PR Test | |
| on: | |
| # Push to main triggers the default GitHub-hosted jobs (cheap CPU/unit and | |
| # agent adapter checks), catching PR-pair regressions where two PRs pass | |
| # individually but main is broken after both land. GPU jobs stay label-gated | |
| # — see each job's `if:` below — so push events never burn the self-hosted | |
| # fleet. | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| types: [opened, reopened, synchronize, labeled] | |
| workflow_dispatch: | |
| inputs: | |
| infinite_run: | |
| description: 'Run training infinitely' | |
| required: false | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| e2e-test-sglang-config: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-sglang-config')) | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_sglang_config.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_sglang_config_distributed.py"}, {"num_gpus": 8, "test_file": "test_sglang_config_mixed_offload.py"}, {"num_gpus": 8, "test_file": "test_sglang_config_mixed_offload_ft.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' | |
| e2e-test-megatron: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-megatron')) | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 4, "test_file": "test_full_disk_weight_update.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo_disaggregate.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen3_0.6B_parallel_check.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}, {"num_gpus": 6, "test_file": "test_qwen3_4B_external_pd.py"}, {"num_gpus": 4, "test_file": "test_qwen2.5_0.5B_fully_async_short.py"}, {"num_gpus": 4, "test_file": "test_qwen2.5_0.5B_fanout_short.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_streaming_partial_rollout.py"}, {"num_gpus": 4, "test_file": "test_qwen3.5_0.8B_gsm8k_short.py"}, {"num_gpus": 4, "test_file": "test_qwen3.5_0.8B_gsm8k_async_short.py"}, {"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--async-save", "test_file": "test_qwen3_4B_ckpt.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' | |
| e2e-test-precision: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-precision')) | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 8, "test_file": "test_qwen3_0.6B_parallel_check.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' | |
| e2e-test-ckpt: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-ckpt')) | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--async-save", "test_file": "test_qwen3_4B_ckpt.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' | |
| cpu-unittest: | |
| if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 0, "test_file": "test_megatron_argument_validation.py"}, {"num_gpus": 0, "test_file": "test_dp_schedule.py"}, {"num_gpus": 0, "test_file": "test_cp_utils.py"}, {"num_gpus": 0, "test_file": "test_metric_report.py"}, {"num_gpus": 0, "test_file": "test_metric_report_dist.py"}, {"num_gpus": 0, "test_file": "test_loss_cp_invariance.py"}, {"num_gpus": 0, "test_file": "test_value_temperature.py"}, {"num_gpus": 0, "test_file": "test_cispo_loss.py"}, {"num_gpus": 0, "test_file": "test_rm_f1.py"}, {"num_gpus": 0, "test_file": "test_rm_gpqa.py"}, {"num_gpus": 0, "test_file": "test_rm_math.py"}, {"num_gpus": 0, "test_file": "test_rm_math_dapo.py"}, {"num_gpus": 0, "test_file": "test_rm_deepscaler.py"}, {"num_gpus": 0, "test_file": "test_sample.py"}, {"num_gpus": 0, "test_file": "test_agent_trajectory.py"}, {"num_gpus": 0, "test_file": "test_rollout_validation.py"}, {"num_gpus": 0, "test_file": "test_placement_group.py"}, {"num_gpus": 0, "test_file": "test_external_sglang_engines.py"}, {"num_gpus": 0, "test_file": "utils/test_hf_checkpoint_saver.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_rollout_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_runtime_hook_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_path_loading_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_generate_contracts.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| pip install torch --index-url https://download.pytorch.org/whl/cpu | |
| pip install pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors | |
| - name: Install | |
| shell: bash | |
| run: cd $GITHUB_WORKSPACE && pip install -e . --no-deps | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| TEST_PATH="${{ matrix.info.test_file }}" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| TEST_ARGS="${{ matrix.info.test_args || '' }}" | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf '%s\n' "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "${{ matrix.info.num_gpus }}" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count ${{ matrix.info.num_gpus }} -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| agent-adapter-test: | |
| if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 0, "test_file": "test_agent_adapters.py"}, {"num_gpus": 0, "test_file": "test_agent_sdk_adapters.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| pip install torch --index-url https://download.pytorch.org/whl/cpu | |
| pip install pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors | |
| pip install openai openai-agents anthropic | |
| - name: Install | |
| shell: bash | |
| run: cd $GITHUB_WORKSPACE && pip install -e . --no-deps | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| TEST_PATH="${{ matrix.info.test_file }}" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| TEST_ARGS="${{ matrix.info.test_args || '' }}" | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf '%s\n' "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "${{ matrix.info.num_gpus }}" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count ${{ matrix.info.num_gpus }} -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| e2e-test-image: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-image')) | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| info: [{"num_gpus": 4, "test_file": "test_full_disk_weight_update.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo_disaggregate.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen3_0.6B_parallel_check.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}, {"num_gpus": 6, "test_file": "test_qwen3_4B_external_pd.py"}, {"num_gpus": 4, "test_file": "test_qwen2.5_0.5B_fully_async_short.py"}, {"num_gpus": 4, "test_file": "test_qwen2.5_0.5B_fanout_short.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_streaming_partial_rollout.py"}, {"num_gpus": 4, "test_file": "test_qwen3.5_0.8B_gsm8k_short.py"}, {"num_gpus": 4, "test_file": "test_qwen3.5_0.8B_gsm8k_async_short.py"}, {"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer gpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer cpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--save-optimizer cpu --load-optimizer gpu", "test_file": "test_qwen3_4B_ckpt.py"}, {"num_gpus": 8, "test_args": "--async-save", "test_file": "test_qwen3_4B_ckpt.py"}] | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime-test:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' | |
| e2e-test-changed-detect: | |
| if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-changed')) | |
| runs-on: self-hosted | |
| outputs: | |
| matrix: ${{ steps.detect.outputs.matrix }} | |
| has_tests: ${{ steps.detect.outputs.has_tests }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect changed tests | |
| id: detect | |
| shell: bash | |
| run: | | |
| CHANGED=$(git diff --name-only --diff-filter=AM origin/main...HEAD -- 'tests/test_*.py' 'tests/plugin_contracts/test_*.py' || true) | |
| if [ -z "$CHANGED" ]; then | |
| echo "No new or modified test files found." | |
| echo "has_tests=false" >> $GITHUB_OUTPUT | |
| echo 'matrix={"info":[]}' >> $GITHUB_OUTPUT | |
| else | |
| echo "Changed test files:" | |
| echo "$CHANGED" | |
| MATRIX="[" | |
| FIRST=true | |
| for filepath in $CHANGED; do | |
| # Extract NUM_GPUS from the test file, default to 8 | |
| NGPU=$(grep -oP '^NUM_GPUS\s*=\s*\K\d+' "$filepath" | head -1) | |
| NGPU=${NGPU:-8} | |
| if [ "$FIRST" = true ]; then FIRST=false; else MATRIX+=","; fi | |
| MATRIX+="{\"test_file\":\"$filepath\",\"num_gpus\":$NGPU}" | |
| done | |
| MATRIX+="]" | |
| echo "has_tests=true" >> $GITHUB_OUTPUT | |
| echo "matrix={\"info\":$MATRIX}" >> $GITHUB_OUTPUT | |
| echo "Generated matrix: $MATRIX" | |
| fi | |
| e2e-test-changed: | |
| needs: e2e-test-changed-detect | |
| if: needs.e2e-test-changed-detect.outputs.has_tests == 'true' | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.e2e-test-changed-detect.outputs.matrix) }} | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} | |
| WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} | |
| SLIME_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }} | |
| SLIME_TEST_USE_DEEPEP: ${{ matrix.info.use_deepep || '0' }} | |
| SLIME_TEST_USE_FP8_ROLLOUT: ${{ matrix.info.use_fp8_rollout || '0' }} | |
| SLIME_TEST_ENABLE_EVAL: ${{ matrix.info.enable_eval || '1' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Execute | |
| shell: bash | |
| run: | | |
| docker run --pull=always --rm \ | |
| --privileged \ | |
| --cap-add SYS_NICE \ | |
| --security-opt seccomp=unconfined \ | |
| --network host \ | |
| --gpus all \ | |
| --ipc=host \ | |
| --shm-size=16g \ | |
| --ulimit memlock=-1 \ | |
| --ulimit stack=67108864 \ | |
| --memory=0 \ | |
| --memory-swap=0 \ | |
| -e http_proxy \ | |
| -e https_proxy \ | |
| -e HTTP_PROXY \ | |
| -e HTTPS_PROXY \ | |
| -e GITHUB_COMMIT_NAME \ | |
| -e WANDB_API_KEY \ | |
| -e SLIME_TEST_ENABLE_INFINITE_RUN \ | |
| -e SLIME_TEST_USE_DEEPEP \ | |
| -e SLIME_TEST_USE_FP8_ROLLOUT \ | |
| -e SLIME_TEST_ENABLE_EVAL \ | |
| -e TEST_FILE="${{ matrix.info.test_file }}" \ | |
| -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \ | |
| -e NUM_GPUS="${{ matrix.info.num_gpus }}" \ | |
| -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ | |
| -v /mnt/nvme0n1/slime_ci:/data/slime_ci \ | |
| -v /mnt/nvme0n1/slime_ci/models:/root/models \ | |
| -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \ | |
| -w "$GITHUB_WORKSPACE" \ | |
| slimerl/slime:latest \ | |
| bash -lc ' | |
| set -euo pipefail | |
| pip install -e . --no-deps --break-system-packages | |
| TEST_PATH="$TEST_FILE" | |
| if [[ "$TEST_PATH" != tests/* ]]; then | |
| TEST_PATH="tests/$TEST_PATH" | |
| fi | |
| if [[ -n "$TEST_ARGS" ]]; then | |
| read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS") | |
| else | |
| TEST_ARGS_ARRAY=() | |
| fi | |
| if [ "$NUM_GPUS" = "0" ]; then | |
| python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| else | |
| python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" | |
| fi | |
| ' |