Drop C-side smear from per-line memory attribution and the flame view

scalene/scalene_memory_profiler.py

@@ -24,7 +24,10 @@
     ScaleneStatistics,
     StackFrame,
 )
-from scalene.scalene_utility import intern_stack_frame
+from scalene.scalene_utility import (
+    intern_stack_frame,
+    line_has_alloc_opcode,
+)
 
 
 class ScaleneMemoryProfiler:
@@ -267,9 +270,46 @@ def process_malloc_free_samples(
             fname = item.filename
             lineno = item.lineno
 
+            # Smear suppression. The C++ interposer stamps every sample
+            # with the leaf user frame from ``whereInPython``, but
+            # CPython internals (arena resizes, GC, scalene's own
+            # bookkeeping under the GIL) emit raw mallocs whose bytes
+            # land on whichever leaf line the eval loop happens to be
+            # on. If that line's bytecode cannot have caused the alloc
+            # (no CALL / BUILD_ / LIST_ / … opcode — see
+            # ``line_has_alloc_opcode``), the sample is "smear".
+            #
+            # Earlier versions redirected smear up the captured stack
+            # to the nearest alloc-capable caller. That puts the bytes
+            # on whatever call site is currently active, which is
+            # misleading: e.g. ``x = doit2(x)`` is a CALL but doit2
+            # never allocates — the bytes really come from heap state
+            # set up by doit1 in a prior frame that's no longer on the
+            # stack. For the same reason, the captured stack itself
+            # isn't trustworthy here: it records which user frames
+            # were active at the moment of the malloc, none of which
+            # caused it. So we drop the sample from every per-line
+            # accumulator *and* from ``memory_stacks``. The bytes are
+            # still counted in the global running footprint (so the
+            # global peak / sparkline are accurate), but no line,
+            # function, or flame-chart path takes the blame.
+            leaf_can_alloc = line_has_alloc_opcode(fname, lineno)
+
+            count = item.count / self.BYTES_PER_MB
+            if not leaf_can_alloc:
+                # Keep the running per-item ``curr`` accurate so the
+                # next legitimate sparkline sample reflects the right
+                # footprint. Everything else (memory_stacks, per-line
+                # accumulators, allocs / last_malloc tracking) skips
+                # this sample.
+                if is_malloc:
+                    curr += count
+                else:
+                    curr -= count
+                continue
+
             # Add the byte index to the set for this line (if it's not there already).
             stats.bytei_map[fname][lineno].add(item.bytecode_index)
-            count = item.count / self.BYTES_PER_MB
             if is_malloc:
                 # Attribute the sampled bytes to the Python call stack
                 # captured *synchronously* in C++ inside process_malloc

scalene/scalene_utility.py

@@ -10,7 +10,7 @@
 import tempfile
 import threading
 import webbrowser
-from types import BuiltinFunctionType, FrameType, FunctionType, ModuleType
+from types import BuiltinFunctionType, CodeType, FrameType, FunctionType, ModuleType
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from scalene.scalene_config import scalene_date, scalene_version
@@ -996,3 +996,138 @@ def wrapped(*args: Any, **kwargs: Any) -> Any:
         if isinstance(attr, (BuiltinFunctionType, FunctionType)):
             wrapped_attr = signal_blocking_wrapper(attr)
             setattr(module, attr_name, wrapped_attr)
+
+
+# --------------------------------------------------------------------------- #
+#  Allocator-opcode detection (used by the smear-suppression path in
+#  scalene_memory_profiler.process_malloc_free_samples)
+# --------------------------------------------------------------------------- #
+#
+# The C++ heap interposer stamps every malloc / free sample with the leaf
+# Python frame from ``PyFrame_GetLineNumber``. When that leaf line's
+# bytecode cannot possibly have caused the allocation (e.g. ``z = z * z``
+# on two floats — only LOAD_FAST / BINARY_OP / STORE_FAST), but raw
+# ``malloc`` *does* fire under the GIL during that line (arena resizes,
+# GC, scalene's own bookkeeping, …), the bytes get smeared onto whichever
+# arithmetic line the eval loop happened to be on. ``line_has_alloc_opcode``
+# is the predicate the memory profiler uses to detect those samples; when
+# it returns False for the leaf the profiler drops the sample from every
+# per-line accumulator (the bytes still flow into the global footprint and
+# into ``memory_stacks``, so the flame view is unaffected).
+
+# Opcode-name prefixes that mark a line as potentially-allocating. CPython
+# guarantees these prefixes are stable across releases (the individual
+# opcodes inside them shift between versions — e.g. CALL_FUNCTION_EX,
+# CALL_KW, CALL_INTRINSIC_1 — but the CLAUDE.md guidance approves
+# ``.startswith("CALL")`` and the same shape applies to the others).
+_ALLOC_OPCODE_PREFIXES: Tuple[str, ...] = (
+    "CALL",        # CALL, CALL_KW, CALL_FUNCTION_EX, CALL_INTRINSIC_*, …
+    "BUILD_",      # BUILD_LIST / TUPLE / SET / MAP / STRING / SLICE / …
+    "LIST_",       # LIST_APPEND, LIST_EXTEND, LIST_TO_TUPLE
+    "SET_",        # SET_ADD, SET_UPDATE
+    "MAP_",        # MAP_ADD
+    "DICT_",       # DICT_UPDATE, DICT_MERGE
+    "IMPORT_",     # IMPORT_NAME / FROM / STAR
+    "MAKE_",       # MAKE_FUNCTION, MAKE_CELL
+    "FORMAT_",     # FORMAT_VALUE, FORMAT_SIMPLE, FORMAT_WITH_SPEC
+)
+
+# Exact opcode names that also count as allocation-capable. Includes
+# container iteration (which may call user ``__iter__`` / ``__next__``),
+# subscript / attr access (which may call ``__getitem__`` / ``__getattr__``
+# / etc.), and various async / exception / class-build entry points.
+# Keeping this list reasonably broad addresses the list-comprehension and
+# container-op case explicitly: a list comp without an inline range() call
+# still has BUILD_LIST, GET_ITER, FOR_ITER, and LIST_APPEND on its line.
+_ALLOC_OPCODE_EXACT: frozenset[str] = frozenset(
+    {
+        "GET_ITER",
+        "GET_YIELD_FROM_ITER",
+        "FOR_ITER",
+        "GET_AWAITABLE",
+        "GET_AITER",
+        "GET_ANEXT",
+        "SEND",
+        "YIELD_VALUE",
+        "YIELD_FROM",
+        "BINARY_SUBSCR",
+        "STORE_SUBSCR",
+        "DELETE_SUBSCR",
+        "LOAD_ATTR",
+        "STORE_ATTR",
+        "DELETE_ATTR",
+        "LOAD_BUILD_CLASS",
+        "RAISE_VARARGS",
+        "RERAISE",
+        "BEFORE_WITH",
+        "BEFORE_ASYNC_WITH",
+        "WITH_EXCEPT_START",
+        "CONVERT_VALUE",
+    }
+)
+
+# Per-file cache: filename → set of source lines containing at least one
+# allocation-capable opcode. ``None`` means the source could not be read or
+# compiled — in that case we trust the leaf attribution (don't redirect).
+_file_alloc_lines_cache: Dict[str, Optional[set[int]]] = {}
+
+
+def _is_alloc_opname(opname: str) -> bool:
+    if opname in _ALLOC_OPCODE_EXACT:
+        return True
+    return any(opname.startswith(prefix) for prefix in _ALLOC_OPCODE_PREFIXES)
+
+
+def _collect_alloc_lines(code: CodeType, out: set[int]) -> None:
+    """Walk a code object (and nested code consts) and record every
+    source line that carries at least one allocation-capable opcode."""
+    import dis  # local import: only used at serialization time
+
+    last_line: Optional[int] = None
+    for instr in dis.get_instructions(code):
+        # Python 3.13+ exposes ``Instruction.line_number`` (int|None on
+        # every instr). Older Pythons only set ``starts_line`` on the
+        # first instr per source line; propagate it forward manually.
+        ln = getattr(instr, "line_number", None)
+        if ln is None:
+            sl = getattr(instr, "starts_line", None)
+            if isinstance(sl, int):
+                last_line = sl
+        else:
+            last_line = ln
+        if last_line is not None and _is_alloc_opname(instr.opname):
+            out.add(last_line)
+    for const in getattr(code, "co_consts", ()):
+        if hasattr(const, "co_code"):
+            _collect_alloc_lines(const, out)
+
+
+def line_has_alloc_opcode(filename: str, lineno: int) -> bool:
+    """Return True if ``filename:lineno`` carries an allocation-capable
+    opcode. Sources that cannot be compiled are treated as "yes" so we
+    leave their attribution alone."""
+    if not filename or lineno <= 0:
+        return False
+    sentinel = _file_alloc_lines_cache
+    cached = _file_alloc_lines_cache.get(filename, sentinel)
+    if cached is sentinel:
+        import linecache  # local: only used at serialization time
+
+        alloc_lines: Optional[set[int]] = set()
+        try:
+            src = linecache.getlines(filename)
+            if src:
+                code = compile("".join(src), filename, "exec")
+                assert alloc_lines is not None
+                _collect_alloc_lines(code, alloc_lines)
+            else:
+                alloc_lines = None
+        except (SyntaxError, ValueError, TypeError, OSError):
+            alloc_lines = None
+        _file_alloc_lines_cache[filename] = alloc_lines
+        cached = alloc_lines
+    if cached is None:
+        return True
+    return lineno in cached
+
+

test/line_attribution_tests/arithmetic_smear.py

@@ -0,0 +1,62 @@
+"""Regression fixture: pure-arithmetic hot loops must not be credited
+with C-side allocation traffic from CPython internals.
+
+The workload alternates two phases per outer iteration:
+
+  * ``do_allocs`` runs three list comprehensions of increasing size.
+    Those lines (15, 16, 17) genuinely allocate via Python bytecode
+    that has CALL (``range``) and ``BUILD_LIST`` / ``LIST_APPEND``
+    on it. They are the legitimate attribution target for bytes.
+
+  * ``hot_arith`` runs a tight ``while`` loop whose body is only
+    ``LOAD_FAST`` / ``BINARY_OP`` / ``STORE_FAST`` — no CALL, no
+    container-building opcode, no way to invoke a C-side allocator
+    from user code. Yet during this loop CPython internals (arena
+    resizes, GC, scalene's own bookkeeping under the GIL) emit raw
+    ``malloc`` calls. Pre-fix, those bytes were credited to whichever
+    arithmetic line the eval loop happened to be on — hundreds of MB
+    of phantom traffic on ``z = z * z``.
+
+Sized so Scalene reliably samples it: ``do_allocs`` retains a few
+hundred MB of transient buffer activity per outer iteration, and the
+outer loop runs long enough that the malloc sampler fires many times.
+"""
+
+
+def do_allocs():
+    """Real allocator work — list comprehensions on lines 15-17 have
+    CALL (``range``) and BUILD_LIST opcodes, so they are eligible
+    targets for byte attribution."""
+    a = [i * i for i in range(0, 100_000)][99_999]   # line 15
+    b = [i * i for i in range(0, 200_000)][199_999]  # line 16
+    c = [i for i in range(0, 300_000)][299_999]      # line 17
+    return a + b + c
+
+
+def hot_arith(x):
+    """Pure float arithmetic — no CALL, no BUILD_*, no allocator
+    opcode on lines 24-27. Any byte attribution that lands on these
+    lines is smear from CPython internals running under the GIL."""
+    z = 0.1
+    i = 0
+    while i < 100_000:
+        z = z * z       # line 24
+        z = x * x       # line 25
+        z = z * z       # line 26
+        z = z * z       # line 27
+        i += 1          # line 28
+    return z
+
+
+def run():
+    x = 1.01
+    for _ in range(9):
+        for _ in range(9):
+            do_allocs()       # line 36 — legitimate alloc caller
+            hot_arith(x)      # line 37 — smear target (caller of the
+                              # pure-arith loop; redirected smear lands here)
+    return x
+
+
+if __name__ == "__main__":
+    run()