mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 04:59:28 +00:00
Marius CiepluchandClaude Opus 5.5
800204de87
Reran benchmarks/harnessx_removal_equality.py record from a git archive export ofb9afb66(tasks 1-6 landed; same venv freeze hash as the baseline, harnessx not importable, no API keys, empty config) and compared it against 2026-09-30-harnessx-removal-baseline: equal=True differences=0 All 13 output files (pairs.json, benchmark, standard scan, improve --dry-run) are raw-byte identical to the baseline. The suite map (suite-map.json) accounts for 29 deleted ids, 2 renamed ids and 83 added ids; each deleted or renamed id cites the design's Tests table and commitcbadc38orb9afb66. Every surviving test id keeps its outcome (1740 passed, 38 skipped incl. the README xfail). Instrument note: a first record failed test_push_hook twice because the recording shell's PATH lacked .venv/bin (the hook execs `ousast`); the same two tests fail atb9afb66without it and pass with it. The committed record has .venv/bin on PATH. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
72 lines
1.9 KiB
JSON
72 lines
1.9 KiB
JSON
{
|
|
"child_env": {
|
|
"OPENULTRASAST_SKIP_DOTENV": "1",
|
|
"PYTHONHASHSEED": "0",
|
|
"api_keys": "every *_API_KEY unset",
|
|
"config": "empty"
|
|
},
|
|
"exclusions": [
|
|
"JSON keys: created_at, finished, generated_at, runtime_seconds, scan_id, started, timestamp",
|
|
"JSON keys ending in: _seconds, _ms",
|
|
"run directory names / run ids (<timestamp>-<8 hex>, benchmark.py create_benchmark_run, run.py create_scan_run) -> <RUN_ID>",
|
|
"trace/events.jsonl as a whole (timings); never copied",
|
|
"the export's absolute root path -> <ROOT>; the children's TMPDIR -> <TMP>",
|
|
"suite: only the outcome per test id is kept (junit durations dropped)",
|
|
"env.json is a provenance record (commit, freeze hash, instrument checks), not compared"
|
|
],
|
|
"freeze_sha256": "0e5ad41948a751e1c60f72738c48f5ac482b24b67e51c77369ab3be51fa418eb",
|
|
"harnessx_importable": false,
|
|
"inputs": {
|
|
"benchmarks/manifests/java-spring-boot-vulnerable.toml": 1032,
|
|
"benchmarks/manifests/python-vulnerable.toml": 2771,
|
|
"benchmarks/pairs/catalog.toml": 6052
|
|
},
|
|
"instrument_checks": {
|
|
"benchmark": {
|
|
"expected": 13,
|
|
"findings": 12,
|
|
"matched": 12,
|
|
"missed": 1
|
|
},
|
|
"benchmark_files": {
|
|
"benchmark_result.json": 5896,
|
|
"calibration_records.json": 406,
|
|
"external_baseline_deltas.json": 3
|
|
},
|
|
"improve": {
|
|
"journal_bytes": null,
|
|
"rounds": 1
|
|
},
|
|
"openultrasast_module": "<ROOT>/src/openultrasast/__init__.py",
|
|
"pairs": {
|
|
"detected_vuln": 57,
|
|
"labeled_expected": 342,
|
|
"labeled_matched": 82,
|
|
"outcomes": 317,
|
|
"pair_correct": 32,
|
|
"pairs": 317
|
|
},
|
|
"scan": {
|
|
"file_targets": 2,
|
|
"findings": 17
|
|
},
|
|
"scan_files": {
|
|
"findings.json": 20469,
|
|
"fusion.json": 6284,
|
|
"harness.json": 189,
|
|
"manifest.json": 13266,
|
|
"report.sarif": 40310,
|
|
"verification.json": 10573
|
|
},
|
|
"suite": {
|
|
"error": 0,
|
|
"failure": 0,
|
|
"passed": 1740,
|
|
"skipped": 38
|
|
}
|
|
},
|
|
"python": "3.11.13",
|
|
"tree_ish": "b9afb662506dbe73de931b666400de5485443c19",
|
|
"tree_sitter": true
|
|
}
|