marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench(harnessx-removal): deterministic baseline without the extra (task 1)

Browse Source 
Equality script (design section 3) records suite outcomes, `ousast pairs --json`,
the python-vulnerable quick benchmark, one standard scan and `improve --dry-run`
from a frozen export with the harnessx extra absent and no model reachable.
Two independent records are byte-identical (determinism.json).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-30 10:42:57 +02:00
co-authored by Claude Opus 5.5
parent 2ed6d9a036
commit ca5caf735f
20 changed files with 22358 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.kiro/specs/harnessx-removal/tasks.md
+1 -1
View File
@@ -4,7 +4,7 @@ Order follows the design's commit sequence (Requirement 5): nothing is deleted b
and passed its fixture test. Every task is a commit gated on the full suite's own exit code, `ruff check`, and passed its fixture test. Every task is a commit gated on the full suite's own exit code, `ruff check`,
`ruff format --check` and `mypy`; the reconciler stays under 500 lines. `ruff format --check` and `mypy`; the reconciler stays under 500 lines.
- [ ] 1. Baseline without the extra - [x] 1. Baseline without the extra
- Equality script (design §3) with its `find_spec("harnessx")` and freeze checks; re-sync `.venv` without the - Equality script (design §3) with its `find_spec("harnessx")` and freeze checks; re-sync `.venv` without the
`harnessx` extra first. `harnessx` extra first.
- Record `benchmarks/measurements/<date>-harnessx-removal-baseline/`: full suite, `ousast pairs --json`, the - Record `benchmarks/measurements/<date>-harnessx-removal-baseline/`: full suite, `ousast pairs --json`, the
benchmarks/harnessx_removal_equality.py
+396
View File
@@ -0,0 +1,396 @@
"""Equality proof for the HarnessX removal (spec harnessx-removal, design section 3, Requirement 3).
``record`` exports a frozen source tree, runs the deterministic commands with no model reachable and the
``harnessx`` extra absent, and stores their normalized outputs. ``compare`` diffs two such records and exits
non-zero on the first difference outside the exclusion list, naming the file and key.
Usage:
python benchmarks/harnessx_removal_equality.py record <out-dir> [--tree-ish HEAD] [--workdir DIR]
python benchmarks/harnessx_removal_equality.py compare <baseline-dir> <candidate-dir> \\
[--suite-map map.json] [--record comparison.json]
The instrument checks come first and fail loudly: ``harnessx`` must not be importable, ``tree_sitter`` must be,
every child must import ``openultrasast`` from the export, every input manifest must be non-empty, every
command must exit 0, and every command must produce non-zero counts (pairs, TPs, expected findings, suite tests).
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any
REPO = Path(__file__).resolve().parents[1]
# Design section 3 "Comparison": timestamps, durations and run identifiers are excluded; nothing else is.
EXCLUDED_KEYS = frozenset({"runtime_seconds", "scan_id", "timestamp", "started", "finished", "created_at", "generated_at"})
EXCLUDED_SUFFIXES = ("_seconds", "_ms")
RUN_ID = re.compile(r"\d{8}T\d{6}Z-[0-9a-f]{8}")
EXCLUSIONS = [
f"JSON keys: {', '.join(sorted(EXCLUDED_KEYS))}",
f"JSON keys ending in: {', '.join(EXCLUDED_SUFFIXES)}",
"run directory names / run ids (<timestamp>-<8 hex>, benchmark.py create_benchmark_run, run.py create_scan_run) -> <RUN_ID>",
"trace/events.jsonl as a whole (timings); never copied",
"the export's absolute root path -> <ROOT>; the children's TMPDIR -> <TMP>",
"suite: only the outcome per test id is kept (junit durations dropped)",
"env.json is a provenance record (commit, freeze hash, instrument checks), not compared",
]
INPUTS = (
"benchmarks/pairs/catalog.toml",
"benchmarks/manifests/python-vulnerable.toml",
"benchmarks/manifests/java-spring-boot-vulnerable.toml",
)
SCAN_KEEP = ("findings.json", "verification.json", "fusion.json", "report.sarif", "harness.json", "manifest.json")
BENCH_KEEP = ("benchmark_result.json", "calibration_records.json", "external_baseline_deltas.json")
class InstrumentError(SystemExit):
def __init__(self, message: str) -> None:
super().__init__(f"instrument check failed: {message}")
def normalize(value: Any, root: str, tmp: str) -> Any:
"""Drop excluded keys recursively and scrub the export root, TMPDIR and run ids from strings."""
if isinstance(value, dict):
return {k: normalize(v, root, tmp) for k, v in value.items() if not _excluded(k)}
if isinstance(value, list):
return [normalize(v, root, tmp) for v in value]
if isinstance(value, str):
return scrub(value, root, tmp)
return value
def _excluded(key: str) -> bool:
return key in EXCLUDED_KEYS or key.endswith(EXCLUDED_SUFFIXES)
def scrub(text: str, root: str, tmp: str) -> str:
text = text.replace(tmp, "<TMP>").replace(root, "<ROOT>")
return RUN_ID.sub("<RUN_ID>", text)
def dump(value: Any) -> str:
return json.dumps(value, indent=1, sort_keys=True) + "\n"
def _child_env(root: Path, tmp: Path) -> dict[str, str]:
env = {k: v for k, v in os.environ.items() if not k.endswith("_API_KEY") and k not in {"PYTHONPATH", "VIRTUAL_ENV"}}
env.update(OPENULTRASAST_SKIP_DOTENV="1", PYTHONPATH=str(root / "src"), TMPDIR=str(tmp), PYTHONHASHSEED="0")
return env
def _run(cmd: list[str], root: Path, env: dict[str, str], label: str) -> str:
proc = subprocess.run(cmd, cwd=root, env=env, capture_output=True, text=True, check=False)
if proc.returncode != 0:
tail = (proc.stdout + proc.stderr)[-2000:]
raise InstrumentError(f"{label} exited {proc.returncode}: {' '.join(cmd)}\n{tail}")
return proc.stdout
def _field(stdout: str, name: str) -> str:
for line in stdout.splitlines():
if line.startswith(f"{name}="):
return line.partition("=")[2]
raise InstrumentError(f"no {name}= line in output")
def _freeze_hash() -> str:
uv = shutil.which("uv")
if uv is None:
raise InstrumentError("uv not found; cannot hash the environment")
out = subprocess.run([uv, "pip", "freeze", "--python", sys.executable], capture_output=True, text=True, check=True).stdout
return hashlib.sha256(out.encode()).hexdigest()
def _suite(xml_path: Path) -> dict[str, str]:
outcomes: dict[str, str] = {}
for case in ET.parse(xml_path).getroot().iter("testcase"):
test_id = f"{case.get('classname')}::{case.get('name')}"
kinds = {child.tag for child in case}
outcome = next((k for k in ("error", "failure", "skipped") if k in kinds), "passed")
outcomes[test_id] = outcome
return dict(sorted(outcomes.items()))
def record(out: Path, tree_ish: str, workdir: Path) -> None:
if importlib.util.find_spec("harnessx") is not None:
raise InstrumentError("harnessx is importable; re-sync the venv without the extra")
if importlib.util.find_spec("tree_sitter") is None:
raise InstrumentError("tree_sitter is not importable; the semantic extra is missing")
workdir.mkdir(parents=True, exist_ok=True)
root = Path(tempfile.mkdtemp(prefix="export-", dir=workdir)).resolve()
tmp = Path(tempfile.mkdtemp(prefix="tmp-", dir=workdir)).resolve()
archive = subprocess.run(["git", "-C", str(REPO), "archive", tree_ish], capture_output=True, check=True).stdout
subprocess.run(["tar", "-x", "-C", str(root)], input=archive, check=True)
env = _child_env(root, tmp)
rs, ts = str(root), str(tmp)
py = sys.executable
out.mkdir(parents=True, exist_ok=True)
checks: dict[str, Any] = {}
module = _run([py, "-c", "import openultrasast; print(openultrasast.__file__)"], root, env, "import probe").strip()
if not module.startswith(rs):
raise InstrumentError(f"openultrasast imported from {module}, not the export {rs}")
checks["openultrasast_module"] = scrub(module, rs, ts)
inputs = {name: (root / name).stat().st_size for name in INPUTS}
if not all(inputs.values()):
raise InstrumentError(f"empty input manifest: {inputs}")
print(f"export={root} module={module} inputs={inputs}", flush=True)
# 1. Full suite, outcome per test id.
junit = tmp / "suite.xml"
_run([py, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--junitxml={junit}"], root, env, "pytest")
suite = _suite(junit)
counts = {k: sum(1 for v in suite.values() if v == k) for k in ("passed", "skipped", "failure", "error")}
if counts["passed"] == 0:
raise InstrumentError("suite recorded no passing tests")
(out / "suite.json").write_text(dump(suite))
checks["suite"] = counts
print(f"suite={counts}", flush=True)
# 2. Pair replay (offline, vendored pairs only).
pairs = normalize(json.loads(_run([py, "-m", "openultrasast.cli", "pairs", "--json"], root, env, "pairs")), rs, ts)
checks["pairs"] = _pair_counts(pairs)
(out / "pairs.json").write_text(dump(pairs))
print(f"pairs={checks['pairs']}", flush=True)
# 3. Quick benchmark replay.
empty = tmp / "empty.toml"
empty.write_text("")
cmd = [
py,
"-m",
"openultrasast.cli",
"benchmark",
"benchmarks/manifests/python-vulnerable.toml",
"--mode",
"quick",
"--config",
str(empty),
]
stdout = _run(cmd, root, env, "benchmark")
expected = int(_field(stdout, "benchmark_expected"))
if expected <= 0:
raise InstrumentError("benchmark expected no findings")
checks["benchmark"] = {k: int(_field(stdout, f"benchmark_{k}")) for k in ("expected", "matched", "missed")}
checks["benchmark"]["findings"] = int(_field(stdout, "findings"))
_keep(Path(_field(stdout, "benchmark_run_dir")), BENCH_KEEP, out / "benchmark", rs, ts, checks, "benchmark_files")
(out / "benchmark" / "stdout.txt").write_text(scrub(stdout, rs, ts))
print(f"benchmark={checks['benchmark']}", flush=True)
# 4. One standard-mode scan (verify + fusion, no model configured).
cmd = [py, "-m", "openultrasast.cli", "scan", "benchmarks/fixtures/python-vulnerable", "--mode", "standard", "--config", str(empty)]
stdout = _run(cmd, root, env, "scan")
checks["scan"] = {"findings": int(_field(stdout, "findings")), "file_targets": int(_field(stdout, "file_targets"))}
if checks["scan"]["file_targets"] <= 0:
raise InstrumentError("standard scan saw no file targets")
_keep(Path(_field(stdout, "run_dir")), SCAN_KEEP, out / "scan", rs, ts, checks, "scan_files")
(out / "scan" / "stdout.txt").write_text(scrub(stdout, rs, ts))
print(f"scan={checks['scan']}", flush=True)
# 5. improve --dry-run; the shim keeps the dry-run journal the CLI writes to a throwaway directory.
shim_out = tmp / "improve"
cmd = [py, str(Path(__file__).resolve()), "_improve", str(shim_out), "benchmarks/manifests/java-spring-boot-vulnerable.toml"]
stdout = _run(cmd, root, env, "improve")
(out / "improve").mkdir(exist_ok=True)
(out / "improve" / "stdout.txt").write_text(scrub(stdout, rs, ts))
# A round with no proposals writes no journal (evolve.py returns before append_round); its absence is
# recorded and compared like any file, so a journal appearing after the removal is a difference.
rounds = [line for line in stdout.splitlines() if line.startswith("round ")]
if not rounds or "rounds=" not in stdout:
raise InstrumentError("improve --dry-run printed no round")
journal = shim_out / "improve_journal.json"
checks["improve"] = {"rounds": len(rounds), "journal_bytes": journal.stat().st_size if journal.is_file() else None}
if journal.is_file():
journal_data = normalize(json.loads(journal.read_text()), rs, ts)
(out / "improve" / "improve_journal.json").write_text(dump(journal_data))
print(f"improve={checks['improve']}", flush=True)
head = subprocess.run(["git", "-C", str(REPO), "rev-parse", tree_ish], capture_output=True, text=True, check=True).stdout.strip()
env_record = {
"tree_ish": head,
"freeze_sha256": _freeze_hash(),
"harnessx_importable": False,
"tree_sitter": True,
"python": sys.version.split()[0],
"child_env": {"OPENULTRASAST_SKIP_DOTENV": "1", "PYTHONHASHSEED": "0", "api_keys": "every *_API_KEY unset", "config": "empty"},
"inputs": inputs,
"instrument_checks": checks,
"exclusions": EXCLUSIONS,
}
(out / "env.json").write_text(dump(env_record))
(out / "README").write_text("Excluded from comparison: " + "; ".join(EXCLUSIONS) + "\n")
shutil.rmtree(root, ignore_errors=True)
shutil.rmtree(tmp, ignore_errors=True)
print(f"recorded {out}")
def _pair_counts(pairs: Any) -> dict[str, int]:
overall = pairs.get("overall", {}) if isinstance(pairs, dict) else {}
counts = {k: int(overall.get(k, 0)) for k in ("pairs", "detected_vuln", "pair_correct", "labeled_expected", "labeled_matched")}
counts["outcomes"] = len(pairs.get("outcomes", [])) if isinstance(pairs, dict) else 0
if counts["pairs"] <= 0 or counts["detected_vuln"] <= 0 or counts["outcomes"] <= 0:
raise InstrumentError(f"pair replay saw {counts}")
return counts
def _keep(run_dir: Path, names: tuple[str, ...], dest: Path, rs: str, ts: str, checks: dict[str, Any], key: str) -> None:
if not run_dir.is_dir():
raise InstrumentError(f"run directory missing: {run_dir}")
dest.mkdir(parents=True, exist_ok=True)
present = {}
for name in names:
source = run_dir / name
if source.is_file():
present[name] = source.stat().st_size
(dest / name).write_text(dump(normalize(json.loads(source.read_text()), rs, ts)))
checks[key] = present
def _improve_shim(out: Path, manifest: str) -> int:
"""Run ``ousast improve --dry-run --no-pair-gate`` and copy the throwaway journal out before cleanup."""
from openultrasast import cli
original = tempfile.TemporaryDirectory
class Keeping(original): # type: ignore[misc,valid-type]
def cleanup(self) -> None:
out.mkdir(parents=True, exist_ok=True)
for name in ("improve_journal.json",):
if (Path(self.name) / name).is_file():
shutil.copy2(Path(self.name) / name, out / name)
super().cleanup()
def __exit__(self, *exc: object) -> None:
self.cleanup()
cli.tempfile.TemporaryDirectory = Keeping # type: ignore[misc]
try:
return int(cli.main(["improve", manifest, "--dry-run", "--no-pair-gate"]) or 0)
finally:
cli.tempfile.TemporaryDirectory = original # type: ignore[misc]
def compare(baseline: Path, candidate: Path, suite_map: dict[str, Any]) -> list[str]:
"""Return every difference outside the exclusion list; empty means equal."""
diffs: list[str] = []
diffs += _compare_suite(
json.loads((baseline / "suite.json").read_text()), json.loads((candidate / "suite.json").read_text()), suite_map
)
files = sorted(
{p.relative_to(baseline).as_posix() for p in baseline.rglob("*") if p.is_file()}
| {p.relative_to(candidate).as_posix() for p in candidate.rglob("*") if p.is_file()}
)
for name in files:
if name in {"suite.json", "env.json", "README", "comparison.json", "determinism.json"}:
continue
a, b = baseline / name, candidate / name
if not a.is_file() or not b.is_file():
diffs.append(f"{name}: present only in {'candidate' if b.is_file() else 'baseline'}")
continue
if a.read_bytes() == b.read_bytes():
continue
if name.endswith((".json", ".sarif")):
diffs.append(f"{name}: {_first_json_diff(json.loads(a.read_text()), json.loads(b.read_text()), '$')}")
else:
diffs.append(f"{name}: first differing line {_first_line_diff(a.read_text(), b.read_text())}")
return diffs
def _compare_suite(a: dict[str, str], b: dict[str, str], suite_map: dict[str, Any]) -> list[str]:
deleted = set(suite_map.get("deleted", ()))
renamed: dict[str, str] = dict(suite_map.get("renamed", {}))
added = set(suite_map.get("added", ())) | set(renamed.values())
diffs = []
for test_id, outcome in a.items():
if test_id in deleted:
continue
target = renamed.get(test_id, test_id)
if target not in b:
diffs.append(f"suite.json: {test_id} missing from candidate")
elif b[target] != outcome:
diffs.append(f"suite.json: {test_id} {outcome} -> {b[target]}")
diffs += [f"suite.json: {t} new in candidate" for t in b if t not in a and t not in added]
return diffs
def _first_json_diff(a: Any, b: Any, path: str) -> str:
if type(a) is not type(b):
return f"{path}: type {type(a).__name__} -> {type(b).__name__}"
if isinstance(a, dict):
for key in sorted(set(a) | set(b)):
if key not in a or key not in b:
return f"{path}.{key}: present only in {'candidate' if key in b else 'baseline'}"
if a[key] != b[key]:
return _first_json_diff(a[key], b[key], f"{path}.{key}")
if isinstance(a, list):
if len(a) != len(b):
return f"{path}: length {len(a)} -> {len(b)}"
for i, (x, y) in enumerate(zip(a, b, strict=True)):
if x != y:
return _first_json_diff(x, y, f"{path}[{i}]")
return f"{path}: {json.dumps(a)[:120]} -> {json.dumps(b)[:120]}"
def _first_line_diff(a: str, b: str) -> str:
la, lb = a.splitlines(), b.splitlines()
for i, (x, y) in enumerate(zip(la, lb, strict=False), 1):
if x != y:
return f"{i}: {x[:120]!r} -> {y[:120]!r}"
return f"line count {len(la)} -> {len(lb)}"
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
rec = sub.add_parser("record")
rec.add_argument("out", type=Path)
rec.add_argument("--tree-ish", default="HEAD", help="git tree-ish to export (a commit, or `git write-tree` for the index)")
rec.add_argument("--workdir", type=Path, default=Path(tempfile.gettempdir()) / "hxr-equality")
cmp_ = sub.add_parser("compare")
cmp_.add_argument("baseline", type=Path)
cmp_.add_argument("candidate", type=Path)
cmp_.add_argument("--suite-map", type=Path, help='JSON {"deleted": [...], "renamed": {old: new}, "added": [...]}')
cmp_.add_argument("--record", type=Path, help="write the comparison result here as JSON")
shim = sub.add_parser("_improve")
shim.add_argument("out", type=Path)
shim.add_argument("manifest")
args = parser.parse_args(argv)
if args.command == "record":
record(args.out.resolve(), args.tree_ish, args.workdir.resolve())
return 0
if args.command == "_improve":
return _improve_shim(args.out, args.manifest)
suite_map = json.loads(args.suite_map.read_text()) if args.suite_map else {}
diffs = compare(args.baseline, args.candidate, suite_map)
if args.record:
args.record.write_text(
dump(
{
"baseline": str(args.baseline),
"candidate": str(args.candidate),
"equal": not diffs,
"differences": diffs,
"exclusions": EXCLUSIONS,
}
)
)
for line in diffs[:20]:
print(line)
print(f"equal={not diffs} differences={len(diffs)}")
return 1 if diffs else 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/README
+1
View File
@@ -0,0 +1 @@
Excluded from comparison: JSON keys: created_at, finished, generated_at, runtime_seconds, scan_id, started, timestamp; JSON keys ending in: _seconds, _ms; run directory names / run ids (<timestamp>-<8 hex>, benchmark.py create_benchmark_run, run.py create_scan_run) -> <RUN_ID>; trace/events.jsonl as a whole (timings); never copied; the export's absolute root path -> <ROOT>; the children's TMPDIR -> <TMP>; suite: only the outcome per test id is kept (junit durations dropped); env.json is a provenance record (commit, freeze hash, instrument checks), not compared
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/benchmark/benchmark_result.json
+181
View File
@@ -0,0 +1,181 @@
{
"baseline_deltas": [],
"benchmark_name": "python-vulnerable",
"benchmark_run_id": "<RUN_ID>",
"calibration_records": [
{
"cwe": "CWE-89",
"evidence": "execute runs a SQL query string built from request input on a prior line",
"failed_stage": "benchmark_ground_truth_matching",
"next_improvement_candidate": "language_hunter",
"path": "app.py",
"reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
"rule_id": null,
"vulnerability_class": "sql injection"
}
],
"false_positives": [],
"language": "python_web",
"metrics": {
"actual_findings_total": 12,
"artifact_links": {
"findings": "<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>/findings.json",
"manifest": "<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>/manifest.json",
"report": "<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>/report.md",
"sarif": "<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>/report.sarif"
},
"expected_findings_total": 13,
"false_positive_findings_total": 0,
"matched_findings_total": 12,
"missed_findings_total": 1,
"model_usage": {},
"per_rule": {
"python-flask-debug": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-os-command": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-path-traversal": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-shell-true": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-sql-injection": {
"false_positives": 0,
"matched": 2,
"missed": 0
},
"python-ssrf": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-ssti": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-unsafe-deserialization": {
"false_positives": 0,
"matched": 2,
"missed": 0
},
"python-unsafe-eval": {
"false_positives": 0,
"matched": 1,
"missed": 0
},
"python-weak-hash": {
"false_positives": 0,
"matched": 1,
"missed": 0
}
},
"precision": 1.0,
"recall": 0.9230769230769231
},
"misses": [
{
"cwe": "CWE-89",
"evidence": "execute runs a SQL query string built from request input on a prior line",
"path": "app.py",
"reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
"rule_id": null,
"vulnerability_class": "sql injection"
}
],
"mode": "quick",
"rule_recommendations": [
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-flask-debug"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-os-command"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-path-traversal"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-shell-true"
},
{
"action": "keep",
"false_positives": 0,
"matched": 2,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-sql-injection"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-ssrf"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-ssti"
},
{
"action": "keep",
"false_positives": 0,
"matched": 2,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-unsafe-deserialization"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-unsafe-eval"
},
{
"action": "keep",
"false_positives": 0,
"matched": 1,
"missed": 0,
"rationale": "rule matched expected findings with no false positives",
"rule_id": "python-weak-hash"
}
],
"scan_run_dir": "<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>"
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/benchmark/calibration_records.json
+12
View File
@@ -0,0 +1,12 @@
[
{
"cwe": "CWE-89",
"evidence": "execute runs a SQL query string built from request input on a prior line",
"failed_stage": "benchmark_ground_truth_matching",
"next_improvement_candidate": "language_hunter",
"path": "app.py",
"reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
"rule_id": null,
"vulnerability_class": "sql injection"
}
]
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/benchmark/external_baseline_deltas.json
+1
View File
@@ -0,0 +1 @@
[]
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/benchmark/stdout.txt
+11
View File
@@ -0,0 +1,11 @@
scan_id=<RUN_ID>
run_dir=<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>
file_targets=2
ranked_targets=2
findings=12
calibrations_applied=0
benchmark_run_id=<RUN_ID>
benchmark_run_dir=<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/benchmarks/<RUN_ID>
benchmark_expected=13
benchmark_matched=12
benchmark_missed=1
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/determinism.json
+16
View File
@@ -0,0 +1,16 @@
{
"baseline": "this directory (record run A)",
"candidate": "record run B: second independent record from a fresh export of the same tree, same venv; not committed",
"differences": [],
"equal": true,
"exclusions": [
"JSON keys: created_at, finished, generated_at, runtime_seconds, scan_id, started, timestamp",
"JSON keys ending in: _seconds, _ms",
"run directory names / run ids (<timestamp>-<8 hex>, benchmark.py create_benchmark_run, run.py create_scan_run) -> <RUN_ID>",
"trace/events.jsonl as a whole (timings); never copied",
"the export's absolute root path -> <ROOT>; the children's TMPDIR -> <TMP>",
"suite: only the outcome per test id is kept (junit durations dropped)",
"env.json is a provenance record (commit, freeze hash, instrument checks), not compared"
],
"raw_bytes_identical": "all 15 files incl. env.json (diff -r empty)"
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/env.json
+71
View File
@@ -0,0 +1,71 @@
{
"child_env": {
"OPENULTRASAST_SKIP_DOTENV": "1",
"PYTHONHASHSEED": "0",
"api_keys": "every *_API_KEY unset",
"config": "empty"
},
"exclusions": [
"JSON keys: created_at, finished, generated_at, runtime_seconds, scan_id, started, timestamp",
"JSON keys ending in: _seconds, _ms",
"run directory names / run ids (<timestamp>-<8 hex>, benchmark.py create_benchmark_run, run.py create_scan_run) -> <RUN_ID>",
"trace/events.jsonl as a whole (timings); never copied",
"the export's absolute root path -> <ROOT>; the children's TMPDIR -> <TMP>",
"suite: only the outcome per test id is kept (junit durations dropped)",
"env.json is a provenance record (commit, freeze hash, instrument checks), not compared"
],
"freeze_sha256": "0e5ad41948a751e1c60f72738c48f5ac482b24b67e51c77369ab3be51fa418eb",
"harnessx_importable": false,
"inputs": {
"benchmarks/manifests/java-spring-boot-vulnerable.toml": 1032,
"benchmarks/manifests/python-vulnerable.toml": 2771,
"benchmarks/pairs/catalog.toml": 6052
},
"instrument_checks": {
"benchmark": {
"expected": 13,
"findings": 12,
"matched": 12,
"missed": 1
},
"benchmark_files": {
"benchmark_result.json": 5891,
"calibration_records.json": 406,
"external_baseline_deltas.json": 3
},
"improve": {
"journal_bytes": null,
"rounds": 1
},
"openultrasast_module": "<ROOT>/src/openultrasast/__init__.py",
"pairs": {
"detected_vuln": 57,
"labeled_expected": 342,
"labeled_matched": 82,
"outcomes": 317,
"pair_correct": 32,
"pairs": 317
},
"scan": {
"file_targets": 2,
"findings": 17
},
"scan_files": {
"findings.json": 20469,
"fusion.json": 6284,
"harness.json": 189,
"manifest.json": 13265,
"report.sarif": 40310,
"verification.json": 10573
},
"suite": {
"error": 0,
"failure": 0,
"passed": 1688,
"skipped": 36
}
},
"python": "3.11.13",
"tree_ish": "ca420bfb99d649f01030b8a28ff3ffa4b32d1b22",
"tree_sitter": true
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/improve/stdout.txt
+5
View File
@@ -0,0 +1,5 @@
manifest=benchmarks/manifests/java-spring-boot-vulnerable.toml
target=<ROOT>/benchmarks/fixtures/java-spring-boot-vulnerable
round 1: no_proposals | recall 100.00%->100.00% fp 0.00%->0.00% score 73->73 | edits: -
rounds=1 accepted=0
dry_run=true (no ledger or mechanism store written)
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/pairs.json
+17216
View File
File diff suppressed because it is too large Load Diff
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/findings.json
+630
View File
@@ -0,0 +1,630 @@
{
"findings": [
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"function_name": null,
"line": 81,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 1.5,
"rationale": "Static pattern python-flask-debug (CWE-489) matched debug on line 81; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"insecure_default",
"syscall_entry"
],
"title": "Flask debug mode enabled needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"function_name": "ping",
"line": 17,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-os-command (CWE-78) matched os.system on line 17; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/ping')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 17,
"function_name": "ping",
"kind": "route",
"line": 15,
"name": "ping",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python OS command execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"function_name": "read",
"line": 55,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-path-traversal (CWE-22) matched open on line 55; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/read')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 55,
"function_name": "read",
"kind": "route",
"line": 54,
"name": "read",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python file open with external input needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"function_name": "run",
"line": 23,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-shell-true (CWE-78) matched subprocess.check_output on line 23; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/run')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 23,
"function_name": "run",
"kind": "route",
"line": 21,
"name": "run",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Subprocess shell execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"function_name": "user",
"line": 44,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-sql-injection (CWE-89) matched .execute on line 44; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/user')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 44,
"function_name": "user",
"kind": "route",
"line": 42,
"name": "user",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python SQL string composition needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"function_name": "report",
"line": 50,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-sql-injection (CWE-89) matched .execute on line 50; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/report')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 50,
"function_name": "report",
"kind": "route",
"line": 48,
"name": "report",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python SQL string composition needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"function_name": "fetch",
"line": 60,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-ssrf (CWE-918) matched requests.get on line 60; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/fetch')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 60,
"function_name": "fetch",
"kind": "route",
"line": 59,
"name": "fetch",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"network_entry",
"syscall_entry"
],
"title": "Python outbound request with external input needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"function_name": "render",
"line": 70,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-ssti (CWE-94) matched render_template_string on line 70; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/render')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 70,
"function_name": "render",
"kind": "route",
"line": 69,
"name": "render",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"injection",
"syscall_entry"
],
"title": "Python server-side template injection needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"function_name": "load",
"line": 33,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-deserialization (CWE-502) matched pickle.loads on line 33; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/load')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 33,
"function_name": "load",
"kind": "route",
"line": 32,
"name": "load",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Unsafe Python deserialization needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"function_name": "config",
"line": 38,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-deserialization (CWE-502) matched yaml.load on line 38; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/config')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 38,
"function_name": "config",
"kind": "route",
"line": 37,
"name": "config",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Unsafe Python deserialization needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"function_name": "calc",
"line": 28,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-unsafe-eval (CWE-95) matched eval on line 28; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/calc')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 28,
"function_name": "calc",
"kind": "route",
"line": 27,
"name": "calc",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Dynamic Python execution needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"function_name": "token",
"line": 65,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 3.8,
"rationale": "Static pattern python-weak-hash (CWE-327) matched md5 on line 65; manual verification still required.",
"reachability_conditions": [],
"reachability_evidence": [
{
"access_declared": false,
"access_evidence": [
"app.get('/token')"
],
"access_level": "public",
"conditions": [],
"defines_handler": true,
"end_line": 65,
"function_name": "token",
"kind": "route",
"line": 64,
"name": "token",
"path": "app.py",
"provenance": "entrypoint:route:public",
"rationale": "route surface classified as public at http_request",
"registered_by": "",
"trust_boundary": "http_request"
}
],
"reachability_status": "reachable",
"rung": "suspicion",
"severity": "medium",
"status": "enabled",
"tags": [
"crypto",
"deserialization",
"filesystem_entry",
"syscall_entry"
],
"title": "Python weak hash needs review"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:app.py:77:execute",
"function_name": null,
"line": 77,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.0,
"rationale": "uninventoried sink execute reached from request",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"overlay-originated"
],
"title": "Overlay coverage: execute"
},
{
"confidence": "medium",
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:safe.py:11:os.system",
"function_name": null,
"line": 11,
"path": "safe.py",
"proposed_snippet": null,
"ranking_priority": 0.0,
"rationale": "uninventoried sink os.system reached from parameter",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "critical",
"status": "enabled",
"tags": [
"overlay-originated"
],
"title": "Overlay coverage: os.system"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:44:identity_constraint",
"function_name": "user",
"line": 44,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on users in user is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local",
"resource:users"
],
"title": "Obligation not discharged: protected_read on users without identity_constraint"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:50:identity_constraint",
"function_name": "report",
"line": 50,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on reports in report is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local",
"resource:reports"
],
"title": "Obligation not discharged: protected_read on reports without identity_constraint"
},
{
"confidence": "low",
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:77:identity_constraint",
"function_name": "search",
"line": 77,
"path": "app.py",
"proposed_snippet": null,
"ranking_priority": 0.3,
"rationale": "protected_read on a resource in search is reached without a dominating identity_constraint. Why the obligation exists: the operation fact alone (function-local mode). Label: function_local.",
"reachability_conditions": [],
"reachability_evidence": [],
"reachability_status": "unknown",
"rung": "suspicion",
"severity": "high",
"status": "enabled",
"tags": [
"obligation:protected_read",
"discharger:identity_constraint",
"obligation_evidence:function_local"
],
"title": "Obligation not discharged: protected_read on a resource without identity_constraint"
}
]
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/fusion.json
+192
View File
@@ -0,0 +1,192 @@
[
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-os-command:app.py:17",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-shell-true:app.py:23",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:44",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=high, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:high"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:50",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=high, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:high"
],
"votes": {
"accepted": 2
},
"warnings": []
},
{
"decision_source": "deterministic-reconciler",
"degradations": [],
"disposition": "accepted",
"finding_id": "python-unsafe-eval:app.py:28",
"model_ids": [],
"panels": [
{
"confidence": 0.75,
"disposition": "accepted",
"false_positive_case": "Reachability or sanitizer evidence could still neutralize the sink.",
"leaning": "vulnerability",
"model_id": null,
"rationale": "reachability=reachable, severity=critical, verified=True",
"role": "panel_a",
"vulnerability_case": "Static corroboration exists and function-level reachability evidence is attached."
},
{
"confidence": 0.6,
"disposition": "accepted",
"false_positive_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"leaning": "false_positive",
"model_id": null,
"rationale": "reachable=True, rejected=False, verified=True",
"role": "panel_b",
"vulnerability_case": "A tainted sink shape exists in the source."
}
],
"triggered": true,
"triggers": [
"gates_risky_fix",
"severity:critical"
],
"votes": {
"accepted": 2
},
"warnings": []
}
]
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/harness.json
+11
View File
@@ -0,0 +1,11 @@
{
"contract_mode": "strict",
"model_roles": {
"hunter": null,
"patcher": null,
"ranker": null,
"verifier": null
},
"processor_versions": {},
"prompt_hashes": {}
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/manifest.json
+461
View File
@@ -0,0 +1,461 @@
{
"artifacts": {
"complexity_map": "complexity_map.json",
"findings": "findings.json",
"fusion": "fusion.json",
"markdown": "report.md",
"overlay": "overlay.json",
"sarif": "report.sarif",
"score": "score.json",
"trajectories": "traces/hunter_trajectories.jsonl",
"verification": "verification.json"
},
"complexity": {
"heuristic_only": true,
"hotspot_count": 14
},
"degradations": [
{
"detail": "no path records; obligations evaluated within entry-point handlers only",
"files": [
"app.py",
"safe.py"
],
"language": "python",
"reason": "obligations_function_local",
"stage": "obligations"
},
{
"detail": "no hunter client or model",
"reason": "intent_adjudication_unavailable",
"stage": "obligations"
},
{
"reason": "cpg_unavailable",
"stage": "model"
},
{
"reason": "hunter_model_unavailable",
"stage": "map"
}
],
"findings": [
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"line": 81,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "rejected"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"line": 17,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"line": 55,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"line": 23,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"line": 44,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"line": 50,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"line": 60,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"line": 70,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"line": 33,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"line": 38,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"line": 28,
"path": "app.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"line": 65,
"path": "app.py",
"rung": "suspicion",
"severity": "medium",
"verification_status": "accepted"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:app.py:77:execute",
"line": 77,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "static_corroboration",
"finding_id": "overlay-coverage:safe.py:11:os.system",
"line": 11,
"path": "safe.py",
"rung": "suspicion",
"severity": "critical",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:44:identity_constraint",
"line": 44,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:50:identity_constraint",
"line": 50,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
},
{
"artifact_refs": {
"findings_json": "findings.json",
"markdown": "report.md",
"sarif": "report.sarif",
"trajectories_jsonl": "traces/hunter_trajectories.jsonl",
"verification_json": "verification.json"
},
"evidence_level": "suspicion",
"finding_id": "obligation:protected_read:app.py:77:identity_constraint",
"line": 77,
"path": "app.py",
"rung": "suspicion",
"severity": "high",
"verification_status": "not_run"
}
],
"fusion": [
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-os-command:app.py:17",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-shell-true:app.py:23",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:44",
"triggers": [
"gates_risky_fix",
"severity:high"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-sql-injection:app.py:50",
"triggers": [
"gates_risky_fix",
"severity:high"
]
},
{
"decision_source": "deterministic-reconciler",
"disposition": "accepted",
"finding_id": "python-unsafe-eval:app.py:28",
"triggers": [
"gates_risky_fix",
"severity:critical"
]
}
],
"model": {
"by_rung": {},
"cost_usd": 0.0,
"coverage": [],
"family_coverage": [],
"model_calls": 0,
"partitions": [],
"query_seconds_by_kind": {},
"question_outcomes": [],
"regions_scanned": 0,
"regions_unasked": 0,
"regions_unjudged": 0,
"scope": null,
"seconds": 0.0,
"skipped": "cpg_unavailable",
"unjudged_sample": []
},
"obligations": {
"degradations": [
{
"detail": "no path records; obligations evaluated within entry-point handlers only",
"files": [
"app.py",
"safe.py"
],
"language": "python",
"reason": "obligations_function_local",
"stage": "obligations"
},
{
"detail": "no hunter client or model",
"reason": "intent_adjudication_unavailable",
"stage": "obligations"
}
],
"findings_by_label": {
"function_local": 3
},
"operations": 4,
"policy_version": null,
"sets": [
{
"handlers": 1,
"module": "app",
"resource": "reports"
},
{
"handlers": 1,
"module": "app",
"resource": "users"
}
],
"sibling_sets": 2,
"under_populated": 2
},
"provenance": {
"agent_commits": 0,
"sampled_commits": 0,
"signals": [
"git:unavailable"
],
"value": "human"
},
"score": {
"by_category": {
"Code Injection": 2,
"Code Quality": 1,
"Command or Argument Injection": 2,
"Cryptographic Issues": 1,
"Directory Traversal": 1,
"Encapsulation": 2,
"Information Leakage": 1,
"SQL Injection": 2
},
"gate": {
"block_severity_reachable": 5,
"blocking": false,
"min_score": 80,
"passed": false
},
"max_severity": 5,
"out_of_scope_dynamic_only": 0,
"penalty_total": 105.6,
"project_score": 17,
"unmapped_cwe": []
},
"stages": {
"completed": [
"static",
"map"
],
"requested": [
"static",
"map"
],
"skipped": []
},
"target": "<ROOT>/benchmarks/fixtures/python-vulnerable"
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/report.sarif
+1076
View File
File diff suppressed because it is too large Load Diff
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/stdout.txt
+6
View File
@@ -0,0 +1,6 @@
scan_id=<RUN_ID>
run_dir=<ROOT>/benchmarks/fixtures/python-vulnerable/.openultrasast/runs/<RUN_ID>
file_targets=2
ranked_targets=2
findings=17
calibrations_applied=1
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/scan/verification.json
+280
View File
@@ -0,0 +1,280 @@
{
"verifications": [
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "No function-level attacker-reachable entry point currently reaches this line.",
"evidence_level": "static_corroboration",
"finding_id": "python-flask-debug:app.py:81",
"pro_case": "A static pattern matched the source line.",
"required_next_step": "Add reachability evidence before prioritizing a fix.",
"status": "rejected",
"tie_breaker": "Provide call-graph, route, CLI, parser, or dynamic evidence linking attacker input to this function.",
"verified": false
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-os-command:app.py:17",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-path-traversal:app.py:55",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-shell-true:app.py:23",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:44",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-sql-injection:app.py:50",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-ssrf:app.py:60",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-ssti:app.py:70",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:33",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-deserialization:app.py:38",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-unsafe-eval:app.py:28",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
},
{
"context_sources": [
"evidence_level",
"finding_id",
"function_name",
"line",
"path",
"reachability_conditions",
"reachability_evidence",
"reachability_status",
"severity",
"tags",
"title"
],
"counter_case": "Impact and exploitability still require deeper verifier or dynamic evidence.",
"evidence_level": "static_corroboration",
"finding_id": "python-weak-hash:app.py:65",
"pro_case": "Static corroboration exists and function-level reachability evidence is attached.",
"required_next_step": "Proceed to standard verifier or dynamic reproduction if severity warrants it.",
"status": "accepted",
"tie_breaker": "Confirm attacker control, sanitizer absence, and impact in the next verification depth.",
"verified": true
}
]
}
benchmarks/measurements/2026-09-30-harnessx-removal-baseline/suite.json
+1726
View File
File diff suppressed because it is too large Load Diff
tests/test_harnessx_removal_equality.py
+65
View File
@@ -0,0 +1,65 @@
"""The equality instrument (harnessx-removal design section 3) sees a difference when there is one.
A comparison that always says "equal" would prove nothing, so these tests feed it records that differ in one
place each and require it to name that place, and records that differ only in excluded fields and require it
to pass.
"""
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
_SPEC = importlib.util.spec_from_file_location("hxr_equality", ROOT / "benchmarks" / "harnessx_removal_equality.py")
assert _SPEC is not None and _SPEC.loader is not None
equality = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(equality)
def _record(path: Path, *, findings: object, suite: dict[str, str], stdout: str = "findings=3\n") -> Path:
(path / "scan").mkdir(parents=True)
(path / "suite.json").write_text(equality.dump(suite))
(path / "scan" / "findings.json").write_text(equality.dump(findings))
(path / "scan" / "stdout.txt").write_text(stdout)
(path / "env.json").write_text(json.dumps({"tree_ish": str(path)}))
return path
def _normalized(raw: dict[str, object], root: str) -> object:
return equality.normalize(raw, root, "/tmp/x")
def test_excluded_fields_and_run_ids_do_not_count(tmp_path: Path) -> None:
a = _normalized({"rule": "sqli", "runtime_seconds": 1.2, "path": "/r1/runs/20260930T101010Z-0123abcd/f.py", "t_ms": 4}, "/r1")
b = _normalized({"rule": "sqli", "runtime_seconds": 9.9, "path": "/r2/runs/20261001T111111Z-ffffffff/f.py", "t_ms": 7}, "/r2")
suite = {"tests.test_x::test_a": "passed"}
diffs = equality.compare(_record(tmp_path / "a", findings=a, suite=suite), _record(tmp_path / "b", findings=b, suite=suite), {})
assert diffs == []
def test_a_changed_finding_is_reported_with_its_key(tmp_path: Path) -> None:
suite = {"tests.test_x::test_a": "passed"}
a = _record(tmp_path / "a", findings=[{"rule": "sqli", "line": 3}], suite=suite)
b = _record(tmp_path / "b", findings=[{"rule": "sqli", "line": 4}], suite=suite)
assert equality.compare(a, b, {}) == ["scan/findings.json: $[0].line: 3 -> 4"]
assert equality.main(["compare", str(a), str(b)]) == 1
def test_a_changed_outcome_or_unmapped_test_is_reported(tmp_path: Path) -> None:
a = _record(tmp_path / "a", findings=[], suite={"t::a": "passed", "t::gone": "passed", "t::old": "skipped"})
b = _record(tmp_path / "b", findings=[], suite={"t::a": "failure", "t::new": "skipped", "t::extra": "passed"}, stdout="findings=2\n")
diffs = equality.compare(a, b, {"deleted": ["t::gone"], "renamed": {"t::old": "t::new"}})
assert diffs == [
"suite.json: t::a passed -> failure",
"suite.json: t::extra new in candidate",
"scan/stdout.txt: first differing line 1: 'findings=3' -> 'findings=2'",
]
def test_a_file_present_on_one_side_only_is_reported(tmp_path: Path) -> None:
a = _record(tmp_path / "a", findings=[], suite={})
b = _record(tmp_path / "b", findings=[], suite={})
(b / "scan" / "fusion.json").write_text("[]\n")
assert equality.compare(a, b, {}) == ["scan/fusion.json: present only in candidate"]