Add pytest plugin (pytest --scalene) — fixes #70
#1027
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| branches: [ master ] | |
| workflow_dispatch: | |
| # Cancel any in-flight runs for the same PR / branch when a new push arrives. | |
| # Master pushes still run to completion (the ``cancel-in-progress`` predicate | |
| # only fires on PR events). | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| run-tests: | |
| runs-on: ${{ matrix.os }} | |
| # 45 min cap: free-threaded 3.13t/3.14t and slow Linux runners can spend | |
| # 25-30 minutes on pytest alone (test_on_off_windows + the memory_* | |
| # tests spawn scalene subprocesses sequentially), leaving very little | |
| # headroom for the parity step that follows. May 12's successful run | |
| # had jobs landing at 24-29 min; a slightly noisier runner pushes them | |
| # over the previous 30 min cap. Bumping to 45 absorbs that variance. | |
| timeout-minutes: 45 | |
| # Free-threaded Python (3.13t/3.14t) is experimental; even when the | |
| # individual pytest / parity steps are flagged continue-on-error | |
| # below, a job-level timeout still reports the matrix entry as | |
| # failed and (with fail-fast) cancels the rest. Mark the whole | |
| # *_t job non-blocking so a t-variant timing out doesn't take the | |
| # rest of the matrix down with it. | |
| continue-on-error: ${{ endsWith(matrix.python, 't') }} | |
| strategy: | |
| # fail-fast: any matrix entry failure cancels the rest. Trades | |
| # full-coverage-on-failure for faster feedback and lower CI burn — | |
| # if Python 3.10 on Ubuntu breaks we want to know within seconds, | |
| # not 30 minutes later after every other entry finishes. | |
| fail-fast: true | |
| matrix: | |
| os: [ ubuntu-latest, macos-latest ] | |
| python: [ '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.13t', '3.14t' ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: select Xcode version | |
| # MacOS > 14.2 requires Xcode >= 15.3; otherwise loading native extension modules fails with e.g.: | |
| # dlopen(/opt/homebrew/lib/python3.11/site-packages/slipcover/probe.abi3.so, 0x0002): bad bind opcode 0x00 | |
| if: startsWith(matrix.os, 'macos-') | |
| run: | | |
| if [ -d /Applications/Xcode_15.3.app/Contents/Developer ]; then sudo xcode-select --switch /Applications/Xcode_15.3.app/Contents/Developer; fi | |
| clang++ --version | |
| g++ --version | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| cache: pip | |
| cache-dependency-path: | | |
| requirements.txt | |
| pyproject.toml | |
| - name: Work around arm64 support on MacOS | |
| # https://github.com/actions/virtual-environments/issues/2557 | |
| if: matrix.os == 'macos-latest' | |
| run: sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/* | |
| - name: Install system build dependencies | |
| # Free-threaded Python may lack pre-built wheels for packages like lxml | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get install -y libxml2-dev libxslt-dev | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements.txt | |
| python -m pip install numpy | |
| - name: Build scalene | |
| run: pip -v install -e . | |
| - name: install test dependencies | |
| run: | | |
| python3 -m pip install pytest pytest-asyncio | |
| # hypothesis ships no wheel for free-threaded CPython 3.13, and | |
| # building it from source fails there: its dependency chain pulls | |
| # pyo3-ffi, and "PyO3 does not support the free-threaded build of | |
| # CPython versions below 3.14". That aborted this step under | |
| # ``bash -e``, so the 3.13t jobs never reached ``run tests`` at all. | |
| # Install it best-effort; the two modules that need it call | |
| # pytest.importorskip("hypothesis") and skip when it's absent. | |
| python3 -m pip install hypothesis || echo "hypothesis unavailable; property-based tests will skip" | |
| # torch/JAX/TensorFlow may not be available on free-threaded Python builds | |
| python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu || true | |
| python3 -m pip install -e ".[test]" || python3 -m pip install -e . | |
| - name: Enable core dumps for free-threaded builds | |
| if: runner.os == 'Linux' && endsWith(matrix.python, 't') | |
| run: | | |
| ulimit -c unlimited | |
| echo '/tmp/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern | |
| # ``apt-get update`` first: the runner image ships a package index | |
| # that can be older than the mirror's contents, and installing gdb | |
| # then 404s on a superseded libc6-dbg. That failed the whole step | |
| # (exit 100), skipping ``run tests`` on 3.14t. gdb is only used by | |
| # the optional "Collect crash backtraces" step, so never let its | |
| # absence stop the job. | |
| sudo apt-get update || true | |
| sudo apt-get install -y gdb || echo "gdb unavailable; crash backtraces will be skipped" | |
| - name: Quick memory profiling smoke test | |
| if: runner.os == 'Linux' && endsWith(matrix.python, 't') | |
| run: | | |
| echo 'x = [i*i for i in range(500000)]' > /tmp/test_ft.py | |
| echo "=== Check PyMem_SetAllocator symbol ===" | |
| nm -D $(python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")/libpython*.so 2>/dev/null | grep PyMem_SetAllocator || echo "Symbol not found in libpython" | |
| echo "=== Check libscalene symbols ===" | |
| nm -D $(python3 -c "import scalene; print(scalene.__path__[0])")/libscalene.so 2>/dev/null | grep "PyMem_\|get_real" || echo "No PyMem symbols in libscalene" | |
| echo "=== Running scalene with memory profiling ===" | |
| LD_PRELOAD=libscalene.so LD_LIBRARY_PATH=$(python3 -c "import scalene; print(scalene.__path__[0])"):$LD_LIBRARY_PATH SCALENE_ALLOCATION_SAMPLING_WINDOW=10485767 gdb -batch -ex 'set follow-fork-mode child' -ex 'set pagination off' -ex run -ex 'thread apply all bt 15' --args python3 -m scalene run --memory -o /tmp/ft_profile.json /tmp/test_ft.py 2>&1 | tail -50 || true | |
| - name: run tests | |
| # Free-threaded Python support is experimental; don't block CI on failures | |
| continue-on-error: ${{ endsWith(matrix.python, 't') }} | |
| # SCALENE_TEST_DIAG=1 makes tests/conftest.py print each skip's reason | |
| # live and flushed. On free-threaded builds the 45-min job timeout can | |
| # cancel pytest before its -rs summary, so this keeps skip reasons in | |
| # the log for future debugging. | |
| env: | |
| SCALENE_TEST_DIAG: ${{ endsWith(matrix.python, 't') && '1' || '' }} | |
| run: | | |
| python3 -m pytest | |
| - name: pytest plugin smoke test | |
| # End-to-end check of the `pytest --scalene` plugin (issue #70) via its | |
| # pytest11 entry point: a bare `pytest --scalene` (no -p) must auto-load | |
| # the plugin, profile the run, and write a profile JSON. This complements | |
| # tests/test_pytest_plugin.py by exercising the installed entry point | |
| # from a clean working directory. | |
| if: runner.os != 'Windows' | |
| continue-on-error: ${{ endsWith(matrix.python, 't') }} | |
| run: | | |
| workdir="$(mktemp -d)" | |
| cat > "$workdir/test_plugin_smoke.py" <<'EOF' | |
| import time | |
| def test_burn_cpu(): | |
| end = time.process_time() + 0.5 | |
| acc = 0 | |
| while time.process_time() < end: | |
| acc = sum(i * i for i in range(2000)) | |
| assert acc >= 0 | |
| EOF | |
| cd "$workdir" | |
| python3 -m pytest --scalene --scalene-outfile "$workdir/smoke-profile.json" test_plugin_smoke.py | |
| test -f "$workdir/smoke-profile.json" | |
| python3 -c "import json,sys; d=json.load(open('$workdir/smoke-profile.json')); assert any('test_plugin_smoke.py' in f for f in d.get('files',{})), 'test file not profiled'; print('plugin smoke test OK')" | |
| - name: Free-threaded parity test | |
| # Runs on ALL matrix entries to verify CPU+memory profiling with | |
| # native code and threads produces comparable results everywhere. | |
| # | |
| # continue-on-error on ALL jobs (not just t-builds): the contention-ratio | |
| # check (8T/1T slowdown <= 3x) is a wall-clock timing assertion, so on a | |
| # noisy shared CI runner it spikes past the threshold (observed 3.1x-5.2x) | |
| # for reasons unrelated to Scalene. Combined with fail-fast, one such | |
| # spike cancelled the whole matrix. The step still runs and prints its | |
| # results for inspection; it just no longer fails the job. | |
| if: runner.os != 'Windows' | |
| continue-on-error: true | |
| run: | | |
| python3 test/test_freethreaded_parity.py | |
| - name: Collect crash backtraces | |
| if: failure() && runner.os == 'Linux' && endsWith(matrix.python, 't') | |
| run: | | |
| for core in /tmp/core.*; do | |
| [ -f "$core" ] || continue | |
| echo "=== Backtrace from $core ===" | |
| gdb -batch -ex "thread apply all bt full" python3 "$core" 2>/dev/null || true | |
| done | |
| # Windows is profiled via threads, not signals, so the signal-based | |
| # interruption that breaks blocking syscalls on Linux (issue #1060) cannot | |
| # happen here. This job runs the Windows side of tests/test_eintr_retry.py -- | |
| # a positive check that profiling a blocking TCP round-trip under | |
| # ``scalene run`` completes cleanly -- guarding against any regression that | |
| # would reintroduce signal-style interruption on Windows. It is a separate, | |
| # scoped job rather than a matrix entry because the full pytest suite is not | |
| # yet Windows-clean (Windows support is partial). | |
| run-tests-windows: | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ '3.11', '3.13' ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| cache: pip | |
| cache-dependency-path: | | |
| requirements.txt | |
| pyproject.toml | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements.txt | |
| python -m pip install numpy | |
| - name: Build scalene | |
| run: pip -v install -e . | |
| - name: Install test dependencies | |
| run: python -m pip install pytest pytest-asyncio hypothesis | |
| - name: Run Windows profiling test | |
| run: python -m pytest tests/test_eintr_retry.py -v |