mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
f09d788568
A census that reports what it found cannot say whether a defect it missed was examined and cleared or never looked at, and that is the difference between a negative and a silence. It is also how a bounded run budget gets sized without guessing: task 13.3 needs to know at what population the declared PHP target enters scope, and asking the record is cheaper than binary-searching with forty-minute scans. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
174 lines
9.3 KiB
Python
174 lines
9.3 KiB
Python
"""Which declared vulnerability families actually establish findings on real code?
|
|
|
|
Seven families are declared. Two are proven end to end: injection on NodeGoat and VAmPI, and access
|
|
control on VAmPI once its change context could complete. The other five have never been checked for
|
|
whether they produce a witness on anything, and a family that is declared but silent is advertised
|
|
coverage the tool does not have.
|
|
|
|
This runs the production discovery and scan over a whole checkout and reports, per family, how many
|
|
regions were asked, how many questions completed, and what was found with the witness and rung that
|
|
established it. It makes no novelty or admission claim: there is no comparison here, only detection.
|
|
|
|
A family reporting nothing is recorded as nothing. That is the point of asking.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from openultrasast.cpg.backend import JoernBackend
|
|
from openultrasast.mapping import analyze_entry_points, php_hook_callbacks
|
|
from openultrasast.model.contracts import ExecutionBudget
|
|
from openultrasast.model.regions import regions_for
|
|
from openultrasast.model.scan import ScanBudget, scan_repository
|
|
from openultrasast.model.shipped import declared_sources
|
|
from openultrasast.preprocess import build_file_target, enumerate_source_files
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
parser.add_argument("--root", type=Path, required=True)
|
|
parser.add_argument("--name", required=True)
|
|
parser.add_argument("--deadline", type=float, default=1800.0)
|
|
parser.add_argument("--max-regions", type=int, default=500)
|
|
args = parser.parse_args()
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
record: dict[str, Any] = {
|
|
"schema_version": 1,
|
|
"experiment": "family-detection-census-v1",
|
|
"subject": args.name,
|
|
"root": str(args.root),
|
|
"claim_scope": "Detection only. No comparison, no novelty, no admission, no capability enabled.",
|
|
}
|
|
target = args.out / f"{args.name}.json"
|
|
|
|
def save() -> None:
|
|
target.write_text(json.dumps(record, indent=2, default=str) + "\n")
|
|
|
|
files = enumerate_source_files(args.root)
|
|
targets = [build_file_target(args.root, path) for path in files]
|
|
opened = sum(path.stat().st_size for path in files if path.is_file())
|
|
regions = regions_for(analyze_entry_points(args.root, targets), targets, shipped=declared_sources(args.root))
|
|
record["inputs"] = {"files": len(files), "bytes": opened, "regions": len(regions)}
|
|
record["regions_by_family"] = dict(collections.Counter(f for r in regions for f in r.families))
|
|
record["languages"] = dict(collections.Counter(r.language for r in regions))
|
|
save()
|
|
if not files or not opened:
|
|
record["error"] = "no readable source"
|
|
save()
|
|
return 2
|
|
|
|
backend = JoernBackend(execution_budget=ExecutionBudget(time.monotonic() + args.deadline, 5.0))
|
|
hooks = php_hook_callbacks(args.root, targets) if any(r.language == "php" for r in regions) else {}
|
|
started = time.monotonic()
|
|
result = scan_repository(
|
|
args.root,
|
|
regions[: args.max_regions],
|
|
backend=backend,
|
|
budget=ScanBudget(max_model_calls=0, max_regions=args.max_regions, order_by_evidence=True),
|
|
execution_budget=ExecutionBudget(time.monotonic() + args.deadline, 5.0),
|
|
ranking_mode="evidence",
|
|
unit="repository",
|
|
population_complete=True,
|
|
)
|
|
record["scan"] = {
|
|
"seconds": round(time.monotonic() - started, 1),
|
|
"build_seconds": result.build_seconds,
|
|
"query_seconds": result.query_seconds,
|
|
"hook_table_hooks": len(hooks),
|
|
"degradations": [str(d.get("reason")) for d in result.degradations],
|
|
# The reason alone cannot be acted on. A partition census that reports files missing carries WHICH
|
|
# files, and without them the gap is a label rather than a lead.
|
|
"degradation_detail": [
|
|
{k: (v[:20] if isinstance(v, list) else v) for k, v in d.items() if k != "stage"} for d in result.degradations
|
|
],
|
|
"questions_total": len(result.question_outcomes),
|
|
"questions_completed": sum(1 for q in result.question_outcomes if q.status == "completed"),
|
|
"findings_total": len(result.findings),
|
|
# Why the outcomes say what they say, counted rather than inferred: a family can find something while
|
|
# every one of its questions is recorded unresolved, and the two numbers then disagree in public.
|
|
"outcome_status": dict(collections.Counter(q.status for q in result.question_outcomes)),
|
|
# WHY an outcome is not completed. Counting only the statuses left three separate causes looking
|
|
# identical, and each one had to be found by reading code rather than by reading the record.
|
|
"outcome_reason": dict(collections.Counter(f"{q.status}:{q.reason}" for q in result.question_outcomes)),
|
|
# A question the scan chose not to arbitrate is recorded as skipped, with why. Without this a cheaper
|
|
# scan and a narrower one look identical, and the whole point of skipping is that it is not the same.
|
|
"deferred": dict(collections.Counter(item.reason for item in (result.scope.deferred if result.scope else ()))),
|
|
}
|
|
|
|
# A family whose every question was pruned asked nothing, and "nothing to ask" is a different statement
|
|
# from "never offered". Counting the skips per family is what keeps those apart in the verdict below.
|
|
deferred_by_family: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter)
|
|
for item in result.scope.deferred if result.scope else ():
|
|
deferred_by_family[item.identity.family][item.reason] += 1
|
|
# Which regions the budget actually reached, in rank order. A census that reports what it found cannot
|
|
# say whether a defect it missed was examined and cleared or never looked at, and that is the difference
|
|
# between a negative and a silence. It is also how a bounded run's budget gets sized without guessing.
|
|
record["selected_paths"] = list(dict.fromkeys(q.identity.path for q in (result.scope.selected if result.scope else ())))[:400]
|
|
asked = collections.Counter(q.identity.family for q in result.question_outcomes)
|
|
completed = collections.Counter(q.identity.family for q in result.question_outcomes if q.status == "completed")
|
|
found: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
|
for finding in result.findings:
|
|
found[finding.family].append(
|
|
{
|
|
"site": finding.site,
|
|
"rung": finding.rung.value,
|
|
"witness": (finding.witness or "")[:200],
|
|
"reached_from": finding.reached_from,
|
|
}
|
|
)
|
|
# Families with regions but no question must appear, or the census hides its own biggest finding: on
|
|
# VAmPI four declared families were offered 25 regions each and asked nothing, and building this list
|
|
# from the questions alone made them invisible rather than negative.
|
|
offered = record["regions_by_family"]
|
|
families = sorted(set(offered) | set(asked) | set(found))
|
|
record["families"] = {
|
|
family: {
|
|
"regions_offering": offered.get(family, 0),
|
|
"questions_asked": asked.get(family, 0),
|
|
"questions_completed": completed.get(family, 0),
|
|
"findings": len(found.get(family, [])),
|
|
"rungs": dict(collections.Counter(item["rung"] for item in found.get(family, []))),
|
|
"examples": found.get(family, [])[:10],
|
|
# The distinction that matters: a family that answered and found nothing is a different
|
|
# statement from one that could not answer, and neither is "clean".
|
|
#
|
|
# Findings are checked FIRST, and that order is the whole point. It used to ask about completion
|
|
# before findings, so a family that established 18 findings on a PHP plugin while no question
|
|
# outcome was marked completed reported "asked but no question completed" -- and a census whose
|
|
# job is to say what each family finds reported nothing for a repository that had found a real
|
|
# SQL injection. A question-outcome count is a statement about bookkeeping; a finding is the
|
|
# thing being censused, and it cannot be hidden behind the other.
|
|
"deferred": dict(deferred_by_family.get(family, {})),
|
|
"verdict": (
|
|
"establishes findings"
|
|
if found.get(family)
|
|
else "no sink of this family in reach"
|
|
if not asked.get(family) and deferred_by_family.get(family, {}).get("tier_zero")
|
|
else "region offered, no question asked"
|
|
if not asked.get(family)
|
|
else "asked but no question completed"
|
|
if not completed.get(family)
|
|
else "answered, nothing found"
|
|
),
|
|
}
|
|
for family in families
|
|
}
|
|
record["summary"] = dict(collections.Counter(entry["verdict"] for entry in record["families"].values()))
|
|
save()
|
|
cleanup = getattr(result, "cleanup", None)
|
|
if callable(cleanup):
|
|
cleanup()
|
|
print(json.dumps({"subject": args.name, "summary": record["summary"]}), flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|