name: run-integration-tests on: pull_request: jobs: changes: runs-on: ubuntu-latest outputs: should_run: ${{ steps.filter.outputs.code }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: | code: - 'audio_separator/**' - 'tests/**' - 'pyproject.toml' - 'poetry.lock' - '.github/workflows/run-integration-tests.yaml' # ── Integration test jobs (parallel across 3 GPU runners) ────────── # # Balanced to ~7 min each so all 3 finish around the same time. # # ensemble-presets (~8 min): test_ensemble_integration (heaviest single file) # core-models (~7 min): test_24bit + test_cli + test_separator_output + roformer tests # stems-and-quality (~6 min): test_ensemble_meaningful + test_multi_stem + test_remote_api ensemble-presets: needs: changes if: needs.changes.outputs.should_run == 'true' runs-on: [self-hosted, linux, gpu] timeout-minutes: 15 env: AUDIO_SEPARATOR_MODEL_DIR: /opt/audio-separator-models steps: - uses: actions/checkout@v4 - name: Verify GPU availability run: nvidia-smi --query-gpu=driver_version,name,memory.total --format=csv,noheader - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install pipx and poetry run: | python -m pip install --user pipx && python -m pipx ensurepath python -m pipx install poetry echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y ffmpeg libsamplerate0 libsamplerate-dev - name: Set up Python with cache uses: actions/setup-python@v5 with: python-version: '3.13' cache: poetry - name: Install Poetry dependencies (GPU) run: poetry install -E gpu - name: Verify pre-cached models run: | MODEL_COUNT=$(ls -1 $AUDIO_SEPARATOR_MODEL_DIR | wc -l) echo "Pre-cached models: $MODEL_COUNT" if [ "$MODEL_COUNT" -lt 10 ]; then echo "::warning::Expected at least 10 pre-cached model files, found $MODEL_COUNT" fi - name: "Run: ensemble preset tests (~8 min)" run: poetry run pytest -sv tests/integration/test_ensemble_integration.py - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: name: ensemble-presets-results path: | *.flac tests/*.flac core-models: needs: changes if: needs.changes.outputs.should_run == 'true' runs-on: [self-hosted, linux, gpu] timeout-minutes: 20 env: AUDIO_SEPARATOR_MODEL_DIR: /opt/audio-separator-models steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install pipx and poetry run: | python -m pip install --user pipx && python -m pipx ensurepath python -m pipx install poetry echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y ffmpeg libsamplerate0 libsamplerate-dev - name: Set up Python with cache uses: actions/setup-python@v5 with: python-version: '3.13' cache: poetry - name: Install Poetry dependencies (GPU) run: poetry install -E gpu - name: "Run: 24-bit, CLI, output, roformer and timing tests (~8 min)" run: | poetry run pytest -sv \ tests/integration/test_24bit_preservation.py \ tests/integration/test_cli_integration.py \ tests/integration/test_separator_output_integration.py \ tests/integration/test_roformer_audio_quality.py \ tests/integration/test_roformer_backward_compatibility.py \ tests/integration/test_roformer_config_validation.py \ tests/integration/test_roformer_e2e.py \ tests/integration/test_roformer_fallback_mechanism.py \ tests/integration/test_roformer_model_switching.py \ tests/integration/test_roformer_new_parameters.py \ tests/integration/test_roformer_inference_timing.py - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: name: core-models-results path: | *.flac tests/*.flac stems-and-quality: needs: changes if: needs.changes.outputs.should_run == 'true' runs-on: [self-hosted, linux, gpu] timeout-minutes: 15 env: AUDIO_SEPARATOR_MODEL_DIR: /opt/audio-separator-models steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install pipx and poetry run: | python -m pip install --user pipx && python -m pipx ensurepath python -m pipx install poetry echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y ffmpeg libsamplerate0 libsamplerate-dev - name: Set up Python with cache uses: actions/setup-python@v5 with: python-version: '3.13' cache: poetry - name: Install Poetry dependencies (GPU) run: poetry install -E gpu - name: "Run: ensemble quality, multi-stem, and remote API tests (~6 min)" run: | poetry run pytest -sv \ tests/integration/test_ensemble_meaningful.py \ tests/integration/test_multi_stem_verification.py \ tests/integration/test_remote_api_integration.py - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: name: stems-and-quality-results path: | *.flac tests/*.flac # ── Windows CPU integration smoke (GitHub-hosted, free) ──────────── # # End-to-end separation on Windows with the CPU provider — one model per # architecture that DirectML supports (RoFormer, MDX, VR). Catches # Windows-specific breakage (paths, soundfile/ffmpeg, model download) # independent of GPU/DirectML. Python 3.12 to match the DirectML jobs # (torch-directml has no 3.13 wheels). # # Models are downloaded on first run and cached with actions/cache # (hosted runners have no pre-baked model dir). windows-cpu-integration: needs: changes if: needs.changes.outputs.should_run == 'true' runs-on: windows-latest timeout-minutes: 45 env: AUDIO_SEPARATOR_MODEL_DIR: C:\audio-separator-models steps: - uses: actions/checkout@v4 - name: Install poetry run: pipx install poetry - name: Set up Python with cache uses: actions/setup-python@v5 with: python-version: '3.12' cache: poetry - name: Install ffmpeg run: choco install ffmpeg -y --no-progress - name: Install Poetry dependencies (CPU) run: poetry install -E cpu - name: Cache separation models uses: actions/cache@v4 with: path: C:\audio-separator-models key: windows-cpu-smoke-models-v1 - name: Ensure model directory exists # Separator raises FileNotFoundError if AUDIO_SEPARATOR_MODEL_DIR # doesn't exist (it never creates it); a cache miss leaves it absent. run: New-Item -ItemType Directory -Force -Path C:\audio-separator-models | Out-Null - name: "Run: one model per DML-relevant architecture (RoFormer, MDX, VR)" run: poetry run pytest -sv tests/integration/test_cli_integration.py -k "mel_band_roformer_karaoke or Inst_HQ_4 or 2_HP-UVR" - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: name: windows-cpu-integration-results path: | *.flac tests/*.flac # ── Windows DirectML integration (self-hosted ephemeral Windows GPU) ── # # Runs on the gha-runner-gpu-windows ephemeral fleet (T4 in WDDM mode; # see docs/CI-GPU-RUNNERS.md). Experimental: continue-on-error until # promoted to a required check. # # torch-directml is layered via pip, NOT the poetry `dml` extra: current # torch-directml releases pin torch==2.4.1, so poetry's lock resolved the # extra to torch-directml 0.1.13 (a torch-1.13-era relic that must not be # used). pip resolves the modern 0.2.5.dev wheel + a compatible torch in # this job's venv only, leaving every other platform's torch untouched. windows-directml: needs: changes if: needs.changes.outputs.should_run == 'true' runs-on: [self-hosted, windows, gpu] # RoFormer/MDX23C run on CPU under DML (allocator limitation) — the # test step alone takes ~30 min. timeout-minutes: 60 continue-on-error: true defaults: run: shell: powershell env: AUDIO_SEPARATOR_MODEL_DIR: C:\audio-separator-models # Short venv base path: the default SYSTEM-profile pypoetry cache dir # plus onnx's very long test-data paths exceeds Windows' 260-char # MAX_PATH and breaks pip ([WinError 206]). POETRY_VIRTUALENVS_PATH: C:\venvs steps: - uses: actions/checkout@v4 - name: Enable Windows long paths run: Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -Value 1 -Type DWord - name: Verify GPU is in WDDM mode (DirectML requirement) run: | nvidia-smi --query-gpu=driver_version,name,driver_model.current --format=csv,noheader $model = nvidia-smi --query-gpu=driver_model.current --format=csv,noheader if ($model -notmatch "WDDM") { throw "GPU not in WDDM mode: $model" } - name: Install Poetry dependencies (base) run: poetry install - name: Layer DirectML packages via pip run: poetry run pip install onnxruntime-directml torch-directml - name: Install VC++ runtime (torch/onnxruntime native DLLs need it) run: | curl.exe -fsSL -o "$env:TEMP\vc_redist.x64.exe" https://aka.ms/vs/17/release/vc_redist.x64.exe Start-Process -FilePath "$env:TEMP\vc_redist.x64.exe" -ArgumentList "/install", "/quiet", "/norestart" -Wait - name: Verify DirectML availability run: | poetry run python -c "import torch_directml; assert torch_directml.is_available(); print('torch_directml OK:', torch_directml.device_name(0))" poetry run python -c "import onnxruntime as ort; ps = ort.get_available_providers(); print(ps); assert 'DmlExecutionProvider' in ps" - name: "Run: DirectML separations (RoFormer, MDX, VR)" run: poetry run pytest -sv tests/integration/test_windows_directml.py - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: name: windows-directml-results path: | *.flac tests/*.flac # ── Gate job for branch protection ──────────────────────────────── integration-test: needs: [changes, ensemble-presets, core-models, stems-and-quality, windows-cpu-integration, windows-directml] if: always() runs-on: ubuntu-latest steps: - name: Check test results run: | if [[ "${{ needs.changes.outputs.should_run }}" != "true" ]]; then echo "Tests skipped - no code changes detected" exit 0 fi echo "ensemble-presets: ${{ needs.ensemble-presets.result }}" echo "core-models: ${{ needs.core-models.result }}" echo "stems-and-quality: ${{ needs.stems-and-quality.result }}" echo "windows-cpu-integration: ${{ needs.windows-cpu-integration.result }}" echo "windows-directml: ${{ needs.windows-directml.result }} (continue-on-error; see job for real outcome)" if [[ "${{ needs.ensemble-presets.result }}" == "failure" ]] || \ [[ "${{ needs.core-models.result }}" == "failure" ]] || \ [[ "${{ needs.stems-and-quality.result }}" == "failure" ]]; then echo "Integration tests failed" exit 1 fi # windows-cpu-integration is reported above but does not fail the # gate yet — it gets promoted to blocking (here + ruleset) after a # stabilization period. See docs/CI-GPU-RUNNERS.md. if [[ "${{ needs.windows-cpu-integration.result }}" == "success" ]]; then echo "All integration tests passed" else echo "GPU integration tests passed; windows-cpu-integration did not succeed (non-blocking)" fi