|
| 1 | +"""Regression test for issue #1086: non-ASCII characters in a profiled file's path. |
| 2 | +
|
| 3 | +Issue: https://github.com/plasma-umass/scalene/issues/1086 |
| 4 | +
|
| 5 | +``pywhere.cpp`` converted ``code->co_filename`` with |
| 6 | +``PyUnicode_AsASCIIString`` on three hot paths (the stack walker used by |
| 7 | +the allocator interposer, ``on_stack``, and the settrace line callback). |
| 8 | +For a perfectly ordinary path like ``.../überschüsse.py`` that conversion |
| 9 | +fails, returning NULL *and leaving a ``UnicodeEncodeError`` set on the |
| 10 | +thread*. Because those paths run from the native allocator hook and from |
| 11 | +trace callbacks, the stray exception surfaced later at an arbitrary, |
| 12 | +unrelated Python line — which is why the reporter saw it land in |
| 13 | +``sysconfig.get_path`` one run and inside pydantic the next. The Python |
| 14 | +side then compounded it by decoding the native sample records as ASCII. |
| 15 | +
|
| 16 | +The fix encodes/decodes filenames as UTF-8 with surrogate escapes and |
| 17 | +NULL-checks every conversion. See ``encodeFilename`` in |
| 18 | +``src/source/pywhere.cpp`` and ``decode_sample`` in |
| 19 | +``scalene/scalene_mapfile.py``. |
| 20 | +
|
| 21 | +Only ``--memory`` runs were affected (the reporter noted ``--cpu-only`` |
| 22 | +worked), since that is what loads the native interposer, so the |
| 23 | +end-to-end test here profiles with memory enabled. |
| 24 | +""" |
| 25 | + |
| 26 | +from __future__ import annotations |
| 27 | + |
| 28 | +import json |
| 29 | +import subprocess |
| 30 | +import sys |
| 31 | +import unicodedata |
| 32 | +from pathlib import Path |
| 33 | + |
| 34 | +import pytest |
| 35 | + |
| 36 | +from scalene.scalene_mapfile import decode_sample |
| 37 | + |
| 38 | +# A module name with two umlauts, as in the issue's reproducer. |
| 39 | +NON_ASCII_MODULE = "überschüsse" |
| 40 | + |
| 41 | +WORKLOAD = """\ |
| 42 | +def f(): |
| 43 | + total = 0 |
| 44 | + data = [] |
| 45 | + for _ in range(300): |
| 46 | + data.append([j for j in range(2000)]) |
| 47 | + total += sum(data[-1]) |
| 48 | + return total |
| 49 | +""" |
| 50 | + |
| 51 | +MAIN = f"""\ |
| 52 | +import {NON_ASCII_MODULE} |
| 53 | +print({NON_ASCII_MODULE}.f()) |
| 54 | +""" |
| 55 | + |
| 56 | +# What WORKLOAD's f() returns: sum(range(2000)) * 300. |
| 57 | +EXPECTED_OUTPUT = str(sum(range(2000)) * 300) |
| 58 | + |
| 59 | + |
| 60 | +def test_decode_sample_round_trips_non_ascii_filenames() -> None: |
| 61 | + """Sample records naming a non-ASCII path must decode, not raise. |
| 62 | +
|
| 63 | + This is the Python half of the fix, and it is deterministic: the old |
| 64 | + ``bytes.decode("ascii")`` raised ``UnicodeDecodeError`` here and killed |
| 65 | + the loop that drains malloc/free samples. |
| 66 | + """ |
| 67 | + path = f"/tmp/pröfile/{NON_ASCII_MODULE}.py" |
| 68 | + record = f"M,1234,4096,0.5,999,0x7f00,{path},7,0" |
| 69 | + assert decode_sample(record.encode("utf-8")) == record |
| 70 | + # Undecodable filesystem bytes reach Python as lone surrogates; those |
| 71 | + # must survive the round trip too rather than raising. |
| 72 | + surrogate_path = "/tmp/\udcff/x.py" |
| 73 | + surrogate_record = f"M,1,1,0.0,1,0x0,{surrogate_path},1,0" |
| 74 | + assert ( |
| 75 | + decode_sample(surrogate_record.encode("utf-8", errors="surrogateescape")) |
| 76 | + == surrogate_record |
| 77 | + ) |
| 78 | + |
| 79 | + |
| 80 | +@pytest.mark.parametrize("in_subdir", [False, True]) |
| 81 | +def test_memory_profile_of_non_ascii_path(tmp_path: Path, in_subdir: bool) -> None: |
| 82 | + """Profiling a program that imports a module at a non-ASCII path works. |
| 83 | +
|
| 84 | + Two placements are covered: the non-ASCII component in the filename |
| 85 | + itself, and in a parent directory (the issue reported both a |
| 86 | + ``überschüsse.py`` module and an ``optionale_verlängerung.py`` under a |
| 87 | + non-ASCII tree). |
| 88 | +
|
| 89 | + The crash assertions run on every attempt — a regression fails them |
| 90 | + immediately and deterministically, since the profiled program dies |
| 91 | + outright. Only the "file shows up in the profile" check is retried, |
| 92 | + against the usual sampling flake (see ``_scalene_subprocess.py``). |
| 93 | + """ |
| 94 | + workdir = tmp_path / ("prögramm" if in_subdir else "program") |
| 95 | + workdir.mkdir() |
| 96 | + module_path = workdir / f"{NON_ASCII_MODULE}.py" |
| 97 | + module_path.write_text(WORKLOAD, encoding="utf-8") |
| 98 | + main_path = workdir / "main.py" |
| 99 | + main_path.write_text(MAIN, encoding="utf-8") |
| 100 | + |
| 101 | + attempts = 3 |
| 102 | + last_files: list = [] |
| 103 | + last_output = "" |
| 104 | + for attempt in range(1, attempts + 1): |
| 105 | + out = tmp_path / f"profile_{attempt}.json" |
| 106 | + try: |
| 107 | + proc = subprocess.run( |
| 108 | + [ |
| 109 | + sys.executable, |
| 110 | + "-m", |
| 111 | + "scalene", |
| 112 | + "run", |
| 113 | + "--memory", |
| 114 | + "--no-browser", |
| 115 | + "-o", |
| 116 | + str(out), |
| 117 | + str(main_path), |
| 118 | + ], |
| 119 | + cwd=str(workdir), |
| 120 | + check=False, |
| 121 | + capture_output=True, |
| 122 | + text=True, |
| 123 | + timeout=300, |
| 124 | + ) |
| 125 | + except subprocess.TimeoutExpired: |
| 126 | + # Scalene startup occasionally wedges under CI contention. |
| 127 | + continue |
| 128 | + last_output = combined = proc.stdout + proc.stderr |
| 129 | + |
| 130 | + assert "UnicodeEncodeError" not in combined, ( |
| 131 | + "Non-ASCII path raised UnicodeEncodeError in the profiled program " |
| 132 | + f"(issue #1086):\n{combined[-3000:]}" |
| 133 | + ) |
| 134 | + assert "UnicodeDecodeError" not in combined, ( |
| 135 | + f"Non-ASCII path raised UnicodeDecodeError:\n{combined[-3000:]}" |
| 136 | + ) |
| 137 | + # The workload prints its result; a crash of the profiled program |
| 138 | + # (the #1086 failure mode) means this never appears. |
| 139 | + assert EXPECTED_OUTPUT in proc.stdout, ( |
| 140 | + f"Profiled program did not run to completion:\n{combined[-3000:]}" |
| 141 | + ) |
| 142 | + assert proc.returncode == 0, ( |
| 143 | + f"scalene exited {proc.returncode}:\n{combined[-3000:]}" |
| 144 | + ) |
| 145 | + |
| 146 | + if not (out.exists() and out.stat().st_size > 0): |
| 147 | + continue |
| 148 | + files = json.loads(out.read_text(encoding="utf-8")).get("files", {}) |
| 149 | + last_files = list(files) |
| 150 | + # The recorded path must come back intact, not mangled or escaped — |
| 151 | + # this exercises the native encode / Python decode pair end to end. |
| 152 | + # Normalize first: a filesystem may hand back the decomposed (NFD) |
| 153 | + # spelling of "ü" regardless of how we wrote it. |
| 154 | + want = unicodedata.normalize("NFC", str(module_path)) |
| 155 | + if any(unicodedata.normalize("NFC", name) == want for name in files): |
| 156 | + return |
| 157 | + pytest.skip( |
| 158 | + "Scalene recorded no samples for the non-ASCII module after " |
| 159 | + f"{attempts} attempts (suspected sampling flake). The program itself " |
| 160 | + f"ran cleanly, so the #1086 crash is not present. " |
| 161 | + f"Profiled files: {last_files}. Last output:\n{last_output[-1000:]}" |
| 162 | + ) |
0 commit comments