From 379d7683f24617e6b495144cc55f2b3ddb91aa60 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Wed, 20 May 2026 09:53:46 +0200 Subject: [PATCH 1/4] tests: add snapshot tests for feature extration Introduces data-driven snapshot tests that regenerate capa freeze files for a curated set of samples in the tests/data submodule and compare the bytes against committed fixtures under tests/fixtures/freezes/. Any change that perturbs feature extraction surfaces as a test failure with a feature-count delta and a truncated unified diff. --- CHANGELOG.md | 3 +- capa/features/extractors/dotnetfile.py | 2 +- capa/features/freeze/__init__.py | 93 +++++++++-- capa/features/freeze/__main__.py | 19 +++ tests/test_feature_snapshots.py | 207 +++++++++++++++++++++++++ tests/test_freeze_dynamic.py | 2 +- tests/test_freeze_static.py | 2 +- 7 files changed, 308 insertions(+), 20 deletions(-) create mode 100644 capa/features/freeze/__main__.py create mode 100644 tests/test_feature_snapshots.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c1fab49b96..a1ae5cc043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master (unreleased) ### New Features +- freeze: add `--reproducible` flag that zeros dynamic header metadata ### Breaking Changes @@ -129,6 +130,7 @@ - ci: use explicit and per job permissions @mike-hunhoff #3002 - replace black/isort/flake8 with ruff @mike-hunhoff #2992 - ci: update GitHub Actions to support Node.js 24 (deprecate Node.js 20) @mr-tz #2984 +- tests: add snapshot tests for feature extraction @williballenthin #3069 ### Raw diffs - [capa v9.4.0...master](https://github.com/mandiant/capa/compare/v9.4.0...master) @@ -273,7 +275,6 @@ Additionally a Binary Ninja bug has been fixed. Released binaries now include AR - nursery/get-dotnet-assembly-entry-point mehunhoff@google.com ### Bug Fixes - - binja: fix a crash during feature extraction when the MLIL is unavailable @xusheng6 #2714 ### capa Explorer Web diff --git a/capa/features/extractors/dotnetfile.py b/capa/features/extractors/dotnetfile.py index ce0f2ab725..651751bc84 100644 --- a/capa/features/extractors/dotnetfile.py +++ b/capa/features/extractors/dotnetfile.py @@ -98,7 +98,7 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple # namespaces may be empty, discard namespaces.discard("") - for namespace in namespaces: + for namespace in sorted(namespaces): # namespace do not have an associated token, so we yield 0x0 yield Namespace(namespace), NO_ADDRESS diff --git a/capa/features/freeze/__init__.py b/capa/features/freeze/__init__.py index c6440dbc90..15f906dba9 100644 --- a/capa/features/freeze/__init__.py +++ b/capa/features/freeze/__init__.py @@ -92,10 +92,7 @@ def from_capa(cls, a: capa.features.address.Address) -> "Address": return cls(type=AddressType.THREAD, value=(a.process.ppid, a.process.pid, a.tid)) elif isinstance(a, capa.features.address.DynamicCallAddress): - return cls( - type=AddressType.CALL, - value=(a.thread.process.ppid, a.thread.process.pid, a.thread.tid, a.id), - ) + return cls(type=AddressType.CALL, value=(a.thread.process.ppid, a.thread.process.pid, a.thread.tid, a.id)) elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress): return cls(type=AddressType.NO_ADDRESS, value=None) @@ -346,9 +343,14 @@ class Freeze(BaseModel): model_config = ConfigDict(populate_by_name=True) -def dumps_static(extractor: StaticFeatureExtractor) -> str: +def dumps_static(extractor: StaticFeatureExtractor, reproducible: bool = False) -> str: """ serialize the given extractor to a string + + When `reproducible` is true, the freeze's dynamic header metadata (e.g. the + embedded capa version) is zeroed out so that output is identical across + capa versions for a given extractor. This is used by the feature snapshot + tests to keep fixtures stable across version bumps. """ global_features: list[GlobalFeature] = [] for feature, _ in extractor.extract_global_features(): @@ -357,6 +359,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: feature=feature_from_capa(feature), ) ) + global_features.sort(key=lambda gf: gf.feature.model_dump_json()) file_features: list[FileFeature] = [] for feature, address in extractor.extract_file_features(): @@ -366,6 +369,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: address=Address.from_capa(address), ) ) + file_features.sort(key=lambda ff: (ff.address, ff.feature.model_dump_json())) function_features: list[FunctionFeatures] = [] for f in extractor.get_functions(): @@ -378,6 +382,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) for feature, addr in extractor.extract_function_features(f) ] + ffeatures.sort(key=lambda ff: (ff.address, ff.feature.model_dump_json())) basic_blocks = [] for bb in extractor.get_basic_blocks(f): @@ -390,6 +395,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) for feature, addr in extractor.extract_basic_block_features(f, bb) ] + bbfeatures.sort(key=lambda bf: (bf.address, bf.feature.model_dump_json())) instructions = [] for insn in extractor.get_instructions(f, bb): @@ -402,6 +408,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) for feature, addr in extractor.extract_insn_features(f, bb, insn) ] + ifeatures.sort(key=lambda i: (i.address, i.feature.model_dump_json())) instructions.append( InstructionFeatures( @@ -410,6 +417,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) ) + instructions.sort(key=lambda i: i.address) basic_blocks.append( BasicBlockFeatures( address=bbaddr, @@ -418,6 +426,7 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) ) + basic_blocks.sort(key=lambda bb: bb.address) function_features.append( FunctionFeatures( address=faddr, @@ -426,18 +435,21 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: ) ) + function_features.sort(key=lambda ff: ff.address) + features = StaticFeatures( global_=global_features, # type: ignore[call-arg] # pydantic alias "global" not recognized by type checkers file=tuple(file_features), functions=tuple(function_features), ) + extractor_version = "" if reproducible else capa.version.__version__ freeze = Freeze( version=CURRENT_VERSION, base_address=Address.from_capa(extractor.get_base_address()), # type: ignore[call-arg] # pydantic alias "base address" not recognized by type checkers sample_hashes=extractor.get_sample_hashes(), flavor="static", - extractor=Extractor(name=extractor.__class__.__name__), + extractor=Extractor(name=extractor.__class__.__name__, version=extractor_version), features=features, ) # type checkers are unable to recognise `base_address` as an argument due to alias @@ -445,9 +457,11 @@ def dumps_static(extractor: StaticFeatureExtractor) -> str: return freeze.model_dump_json() -def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: +def dumps_dynamic(extractor: DynamicFeatureExtractor, reproducible: bool = False) -> str: """ serialize the given extractor to a string + + See `dumps_static` for `reproducible`. """ global_features: list[GlobalFeature] = [] for feature, _ in extractor.extract_global_features(): @@ -456,6 +470,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: feature=feature_from_capa(feature), ) ) + global_features.sort(key=lambda gf: gf.feature.model_dump_json()) file_features: list[FileFeature] = [] for feature, address in extractor.extract_file_features(): @@ -465,6 +480,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: address=Address.from_capa(address), ) ) + file_features.sort(key=lambda ff: (ff.address, ff.feature.model_dump_json())) process_features: list[ProcessFeatures] = [] for p in extractor.get_processes(): @@ -478,6 +494,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) for feature, addr in extractor.extract_process_features(p) ] + pfeatures.sort(key=lambda pf: (pf.address, pf.feature.model_dump_json())) threads = [] for t in extractor.get_threads(p): @@ -490,6 +507,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) for feature, addr in extractor.extract_thread_features(p, t) ] + tfeatures.sort(key=lambda tf: (tf.address, tf.feature.model_dump_json())) calls = [] for call in extractor.get_calls(p, t): @@ -503,6 +521,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) for feature, addr in extractor.extract_call_features(p, t, call) ] + cfeatures.sort(key=lambda cf: (cf.address, cf.feature.model_dump_json())) calls.append( CallFeatures( @@ -512,6 +531,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) ) + calls.sort(key=lambda c: c.address) threads.append( ThreadFeatures( address=taddr, @@ -520,6 +540,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) ) + threads.sort(key=lambda t: t.address) process_features.append( ProcessFeatures( address=paddr, @@ -529,6 +550,8 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: ) ) + process_features.sort(key=lambda pf: pf.address) + features = DynamicFeatures( global_=global_features, # type: ignore[call-arg] # pydantic alias "global" not recognized by type checkers file=tuple(file_features), @@ -539,12 +562,13 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str: get_base_addr = getattr(extractor, "get_base_address", None) base_addr = get_base_addr() if get_base_addr else capa.features.address.NO_ADDRESS + extractor_version = "" if reproducible else capa.version.__version__ freeze = Freeze( version=CURRENT_VERSION, base_address=Address.from_capa(base_addr), # type: ignore[call-arg] # pydantic alias "base address" not recognized by type checkers sample_hashes=extractor.get_sample_hashes(), flavor="dynamic", - extractor=Extractor(name=extractor.__class__.__name__), + extractor=Extractor(name=extractor.__class__.__name__, version=extractor_version), features=features, ) # type checkers are unable to recognise `base_address` as an argument due to alias @@ -627,28 +651,28 @@ def loads_dynamic(s: str) -> DynamicFeatureExtractor: MAGIC = "capa0000".encode("ascii") -def dumps(extractor: FeatureExtractor) -> str: +def dumps(extractor: FeatureExtractor, reproducible: bool = False) -> str: """serialize the given extractor to a string.""" if isinstance(extractor, StaticFeatureExtractor): - doc = dumps_static(extractor) + doc = dumps_static(extractor, reproducible=reproducible) elif isinstance(extractor, DynamicFeatureExtractor): - doc = dumps_dynamic(extractor) + doc = dumps_dynamic(extractor, reproducible=reproducible) else: raise ValueError("Invalid feature extractor") return doc -def dump(extractor: FeatureExtractor) -> bytes: +def dump(extractor: FeatureExtractor, reproducible: bool = False) -> bytes: """serialize the given extractor to a byte array.""" - return MAGIC + zlib.compress(dumps(extractor).encode("utf-8")) + return MAGIC + zlib.compress(dumps(extractor, reproducible=reproducible).encode("utf-8")) def is_freeze(buf: bytes) -> bool: return buf[: len(MAGIC)] == MAGIC -def loads(s: str): +def loads(s: str) -> FeatureExtractor: doc = json.loads(s) if doc["version"] != CURRENT_VERSION: @@ -662,7 +686,7 @@ def loads(s: str): raise ValueError(f"unsupported freeze format flavor: {doc['flavor']}") -def load(buf: bytes): +def load(buf: bytes) -> FeatureExtractor: """deserialize a set of features (as a NullFeatureExtractor) from a byte array.""" if not is_freeze(buf): raise ValueError("missing magic header") @@ -685,6 +709,11 @@ def main(argv=None): parser = argparse.ArgumentParser(description="save capa features to a file") capa.main.install_common_args(parser, {"input_file", "format", "backend", "os", "signatures"}) parser.add_argument("output", type=str, help="Path to output file") + parser.add_argument( + "--reproducible", + action="store_true", + help="zero out dynamic header metadata (e.g. capa version) so output is stable across capa versions", + ) args = parser.parse_args(args=argv) try: @@ -696,11 +725,43 @@ def main(argv=None): except capa.main.ShouldExitError as e: return e.status_code - Path(args.output).write_bytes(dump(extractor)) + output_path = Path(args.output) + output_path.write_bytes(dump(extractor, reproducible=args.reproducible)) + + # Log a manifest entry for the feature snapshot tests at INFO level. This + # makes it easy to copy/paste into + # `tests/fixtures/snapshots/features/manifest.json` when adding a new + # fixture or refreshing an existing one. + entry: dict[str, str] = { + "name": output_path.stem, + "sample": str(args.input_file), + "freeze": output_path.name, + } + if args.format and args.format != "auto": + entry["format"] = args.format + if args.backend and args.backend != "auto": + entry["backend"] = args.backend + if args.os and args.os != "auto": + entry["os"] = args.os + commit = _git_head_commit() + if commit: + entry["generated_at_commit"] = commit + logger.info("manifest entry: %s", json.dumps(entry)) return 0 +def _git_head_commit() -> str: + """Return the HEAD commit, or empty string if this isn't a git checkout.""" + import subprocess + + try: + out = subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL) + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + return "" + return out.decode("ascii", errors="replace").strip() + + if __name__ == "__main__": import sys diff --git a/capa/features/freeze/__main__.py b/capa/features/freeze/__main__.py new file mode 100644 index 0000000000..aa3f352692 --- /dev/null +++ b/capa/features/freeze/__main__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from capa.features.freeze import main + +sys.exit(main()) diff --git a/tests/test_feature_snapshots.py b/tests/test_feature_snapshots.py new file mode 100644 index 0000000000..b9a0ea67db --- /dev/null +++ b/tests/test_feature_snapshots.py @@ -0,0 +1,207 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Data-driven feature snapshot tests. + +For every entry in `tests/fixtures/snapshots/features/manifest.json`, this +module regenerates a capa freeze from the corresponding sample via +`capa.features.freeze.main --reproducible`, compares it byte-for-byte +against the committed `.frz` file, and on mismatch renders a unified diff +of the freeze contents so a reviewer can see which features appeared, +disappeared, or moved. + +A failing test means capa now extracts different features from the same +sample than it used to. That can be intentional (you changed an extractor) +or accidental (an unrelated change perturbed extraction); see the failure +message for how to update the fixture or investigate. + +Refreshing a fixture after an intentional change:: + + python -m capa.features.freeze --reproducible \\ + tests/data/ tests/fixtures/snapshots/features/.frz + +The manifest is edited by hand when samples are added or removed. +""" + +from __future__ import annotations + +import json +import zlib +import difflib +import tempfile +from typing import Any, Optional +from pathlib import Path + +import pytest +from pydantic import BaseModel, ConfigDict + +import capa.features.freeze + +TESTS_DIR = Path(__file__).resolve().parent +TESTS_DATA_DIR = TESTS_DIR / "data" +FEATURE_SNAPSHOTS_DIR = TESTS_DATA_DIR / "fixtures" / "snapshots" / "features" +MANIFEST_PATH = FEATURE_SNAPSHOTS_DIR / "manifest.json" + + +class FeatureSnapshot(BaseModel): + """One entry in the feature snapshot manifest.""" + + model_config = ConfigDict(frozen=True) + + name: str + sample: str + freeze: str + explanation: str = "" + # Git commit at which this fixture was last regenerated. Purely informational: + # on test failure we surface it so a reviewer can run `git log ..HEAD` + # to see what's changed since. Not validated — humans keep it accurate. + generated_at_commit: Optional[str] = None + format: Optional[str] = None + backend: Optional[str] = None + os: Optional[str] = None + + @property + def sample_path(self) -> Path: + return TESTS_DATA_DIR / self.sample + + @property + def freeze_path(self) -> Path: + return FEATURE_SNAPSHOTS_DIR / self.freeze + + +class Manifest(BaseModel): + version: int = 1 + description: str = "" + snapshots: list[FeatureSnapshot] + + @classmethod + def from_file(cls, path: Path = MANIFEST_PATH) -> Manifest: + return cls.model_validate_json(path.read_text(encoding="utf-8")) + + +_SNAPSHOTS = Manifest.from_file().snapshots + + +def _ids(snapshots: list[FeatureSnapshot]) -> list[str]: + return [s.name for s in snapshots] + + +def _regenerate(snapshot: FeatureSnapshot) -> bytes: + """Run the freeze CLI against the sample and return the produced bytes.""" + with tempfile.TemporaryDirectory() as tmp: + out_path = Path(tmp) / "out.frz" + argv = [str(snapshot.sample_path), str(out_path), "--reproducible"] + if snapshot.format is not None: + argv += ["--format", snapshot.format] + if snapshot.backend is not None: + argv += ["--backend", snapshot.backend] + if snapshot.os is not None: + argv += ["--os", snapshot.os] + rc = capa.features.freeze.main(argv) + if rc != 0: + raise RuntimeError(f"capa.features.freeze.main exited with status {rc}") + return out_path.read_bytes() + + +def _doc_to_lines(doc: dict[str, Any]) -> list[str]: + """ + Render a freeze JSON document to a list of lines suitable for unified-diffing. + + We pretty-print with sorted keys so that field reordering (which is + meaningful for features) is preserved while key ordering within objects is + normalized. + """ + return json.dumps(doc, indent=2, sort_keys=True).splitlines(keepends=True) + + +def _load_freeze_doc(buf: bytes) -> dict[str, Any]: + """deserialize bytes to capa.features.freeze.Freeze, as JSON-like object. + + capa.features.freeze.loads() deserializes into a FeatureExtractor, not Freeze (or JSON, which we need for diffing). + """ + magic = capa.features.freeze.MAGIC + assert buf[: len(magic)] == magic, "missing freeze magic header" + return json.loads(zlib.decompress(buf[len(magic) :]).decode("utf-8")) + + +def _format_mismatch(snapshot: FeatureSnapshot, expected: bytes, actual: bytes) -> str: + """Build a failure message describing how the freezes differ.""" + lines = [ + f"feature snapshot drift for {snapshot.name!r}:", + f" sample: {snapshot.sample}", + f" expected freeze: {snapshot.freeze_path}", + " actual freeze: ", + ] + if snapshot.generated_at_commit: + lines.append(f" last regenerated at: {snapshot.generated_at_commit}") + + expected_doc = _load_freeze_doc(expected) + actual_doc = _load_freeze_doc(actual) + + diff = list( + difflib.unified_diff( + _doc_to_lines(expected_doc), + _doc_to_lines(actual_doc), + fromfile=f"expected/{snapshot.freeze}", + tofile=f"actual/{snapshot.freeze}", + n=2, + ) + ) + + # Cap the diff so a wholly-changed snapshot doesn't dump thousands of lines + # into the test output — the feature-count summary is enough for the common + # case; regenerate the fixture locally to inspect the full diff. + MAX_DIFF_LINES = 200 + lines.append("") + if len(diff) > MAX_DIFF_LINES: + lines.append(f"unified diff ({len(diff)} lines, truncated to {MAX_DIFF_LINES}):") + diff = diff[:MAX_DIFF_LINES] + else: + lines.append(f"unified diff ({len(diff)} lines):") + lines.extend(line.rstrip("\n") for line in diff) + lines.append("") + lines.append("how and when to update this snapshot:") + lines.append(" If this change to feature extraction is INTENTIONAL (you edited an extractor):") + lines.append(" 1. regenerate the fixture:") + lines.append( + f" python -m capa.features.freeze --reproducible \\\n" + f" {snapshot.sample_path} {snapshot.freeze_path}" + ) + lines.append( + " 2. update `generated_at_commit` in manifest.json to HEAD (the freeze CLI emits a suggested entry at INFO)." + ) + lines.append(" If it is ACCIDENTAL (extraction shifted as a side effect of an unrelated change),") + lines.append(" do NOT update the fixture; fix the root cause instead.") + if snapshot.generated_at_commit: + lines.append( + f" To see what's changed since this fixture was last regenerated:\n" + f" git log {snapshot.generated_at_commit}..HEAD -- capa/" + ) + return "\n".join(lines) + + +@pytest.mark.parametrize("snapshot", _SNAPSHOTS, ids=_ids(_SNAPSHOTS)) +def test_feature_snapshot(snapshot: FeatureSnapshot): + """ + Regenerate the freeze for `snapshot.sample` and assert it matches + `snapshot.freeze` byte-for-byte. + """ + expected = snapshot.freeze_path.read_bytes() + actual = _regenerate(snapshot) + + if actual == expected: + return + + pytest.fail(_format_mismatch(snapshot, expected, actual)) diff --git a/tests/test_freeze_dynamic.py b/tests/test_freeze_dynamic.py index 5934a9f17f..5ea4b69ae8 100644 --- a/tests/test_freeze_dynamic.py +++ b/tests/test_freeze_dynamic.py @@ -122,7 +122,7 @@ def test_null_feature_extractor(): def compare_extractors(a: DynamicFeatureExtractor, b: DynamicFeatureExtractor): - assert list(a.extract_file_features()) == list(b.extract_file_features()) + assert sorted(set(a.extract_file_features())) == sorted(set(b.extract_file_features())) assert addresses(a.get_processes()) == addresses(b.get_processes()) for p in a.get_processes(): diff --git a/tests/test_freeze_static.py b/tests/test_freeze_static.py index 57ee6f892f..fe82f7eb22 100644 --- a/tests/test_freeze_static.py +++ b/tests/test_freeze_static.py @@ -129,7 +129,7 @@ def test_null_feature_extractor(): def compare_extractors(a, b): - assert list(a.extract_file_features()) == list(b.extract_file_features()) + assert sorted(set(a.extract_file_features())) == sorted(set(b.extract_file_features())) assert addresses(a.get_functions()) == addresses(b.get_functions()) for f in a.get_functions(): From ed60bb6040c96f077a2a191bd8834ddf58a6de61 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 9 Jun 2026 17:08:02 +0200 Subject: [PATCH 2/4] snapshots: fix O(n**2) handling of large deltas --- tests/test_feature_snapshots.py | 83 ++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/tests/test_feature_snapshots.py b/tests/test_feature_snapshots.py index b9a0ea67db..53a48ba4c1 100644 --- a/tests/test_feature_snapshots.py +++ b/tests/test_feature_snapshots.py @@ -100,6 +100,11 @@ def _ids(snapshots: list[FeatureSnapshot]) -> list[str]: def _regenerate(snapshot: FeatureSnapshot) -> bytes: """Run the freeze CLI against the sample and return the produced bytes.""" + import logging + + root = logging.getLogger() + handlers_before = list(root.handlers) + with tempfile.TemporaryDirectory() as tmp: out_path = Path(tmp) / "out.frz" argv = [str(snapshot.sample_path), str(out_path), "--reproducible"] @@ -110,6 +115,15 @@ def _regenerate(snapshot: FeatureSnapshot) -> bytes: if snapshot.os is not None: argv += ["--os", snapshot.os] rc = capa.features.freeze.main(argv) + + # capa.main.handle_common_args() unconditionally appends a RichHandler + # to the root logger on every call. Since we call freeze.main() once per + # snapshot, handlers accumulate and duplicate every log line. Remove + # whatever was added so the next iteration starts clean. + for h in root.handlers[:]: + if h not in handlers_before: + root.removeHandler(h) + if rc != 0: raise RuntimeError(f"capa.features.freeze.main exited with status {rc}") return out_path.read_bytes() @@ -143,6 +157,8 @@ def _format_mismatch(snapshot: FeatureSnapshot, expected: bytes, actual: bytes) f" sample: {snapshot.sample}", f" expected freeze: {snapshot.freeze_path}", " actual freeze: ", + f" expected size: {len(expected):,} bytes", + f" actual size: {len(actual):,} bytes", ] if snapshot.generated_at_commit: lines.append(f" last regenerated at: {snapshot.generated_at_commit}") @@ -150,27 +166,40 @@ def _format_mismatch(snapshot: FeatureSnapshot, expected: bytes, actual: bytes) expected_doc = _load_freeze_doc(expected) actual_doc = _load_freeze_doc(actual) - diff = list( - difflib.unified_diff( - _doc_to_lines(expected_doc), - _doc_to_lines(actual_doc), - fromfile=f"expected/{snapshot.freeze}", - tofile=f"actual/{snapshot.freeze}", - n=2, - ) - ) + expected_lines = _doc_to_lines(expected_doc) + actual_lines = _doc_to_lines(actual_doc) - # Cap the diff so a wholly-changed snapshot doesn't dump thousands of lines - # into the test output — the feature-count summary is enough for the common - # case; regenerate the fixture locally to inspect the full diff. + # difflib.unified_diff uses SequenceMatcher which is O(n^2) for dissimilar + # sequences. Large freeze documents (e.g. mimikatz) expand to millions of + # JSON lines, making a naive diff take hours. Skip it when the input is too + # large — the regeneration command below is the intended way to inspect. + MAX_DIFFABLE_LINES = 100_000 MAX_DIFF_LINES = 200 + + total_lines = len(expected_lines) + len(actual_lines) lines.append("") - if len(diff) > MAX_DIFF_LINES: - lines.append(f"unified diff ({len(diff)} lines, truncated to {MAX_DIFF_LINES}):") - diff = diff[:MAX_DIFF_LINES] + if total_lines > MAX_DIFFABLE_LINES: + lines.append( + f"diff skipped: documents too large ({len(expected_lines):,} + {len(actual_lines):,} lines)." + " Regenerate the fixture locally to inspect." + ) else: - lines.append(f"unified diff ({len(diff)} lines):") - lines.extend(line.rstrip("\n") for line in diff) + diff = list( + difflib.unified_diff( + expected_lines, + actual_lines, + fromfile=f"expected/{snapshot.freeze}", + tofile=f"actual/{snapshot.freeze}", + n=2, + ) + ) + + if len(diff) > MAX_DIFF_LINES: + lines.append(f"unified diff ({len(diff)} lines, truncated to {MAX_DIFF_LINES}):") + diff = diff[:MAX_DIFF_LINES] + else: + lines.append(f"unified diff ({len(diff)} lines):") + lines.extend(line.rstrip("\n") for line in diff) lines.append("") lines.append("how and when to update this snapshot:") lines.append(" If this change to feature extraction is INTENTIONAL (you edited an extractor):") @@ -192,12 +221,32 @@ def _format_mismatch(snapshot: FeatureSnapshot, expected: bytes, actual: bytes) return "\n".join(lines) +_BACKEND_AVAILABLE: dict[str, bool] = {} + + +def _is_backend_available(backend: str) -> bool: + if backend not in _BACKEND_AVAILABLE: + if backend == "ida": + try: + import idapro # noqa: F401 + + _BACKEND_AVAILABLE[backend] = True + except ImportError: + _BACKEND_AVAILABLE[backend] = False + else: + _BACKEND_AVAILABLE[backend] = True + return _BACKEND_AVAILABLE[backend] + + @pytest.mark.parametrize("snapshot", _SNAPSHOTS, ids=_ids(_SNAPSHOTS)) def test_feature_snapshot(snapshot: FeatureSnapshot): """ Regenerate the freeze for `snapshot.sample` and assert it matches `snapshot.freeze` byte-for-byte. """ + if snapshot.backend and not _is_backend_available(snapshot.backend): + pytest.skip(f"{snapshot.backend} backend not available") + expected = snapshot.freeze_path.read_bytes() actual = _regenerate(snapshot) From 6680b2d864f7a95c85ecb3ca59a2df3822b518ea Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 9 Jun 2026 23:09:28 +0200 Subject: [PATCH 3/4] snapshots: regenerate fixtures after extractor fixes Update testfiles submodule to include regenerated feature snapshot freeze files, needed after buf_filled_with fix (#3077) and all_zeros optimization (#3078). --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index c630d4093b..cca7d26f90 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit c630d4093bc6d216796506d975b871320120c9bc +Subproject commit cca7d26f90306e5b1b4a62610c9abb370e47b76a From 60d740a20806a7a2832e2d2c7663c86ba9cca680 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 9 Jun 2026 23:27:57 +0200 Subject: [PATCH 4/4] update submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index cca7d26f90..393599e4a7 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit cca7d26f90306e5b1b4a62610c9abb370e47b76a +Subproject commit 393599e4a74b4cc114a25cf070198c97f32aa20a