diff --git a/.gitignore b/.gitignore index c0baa61..e24654d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,8 @@ vector-store/ *.log .DS_Store + +# Joern writes a `workspace/` beside its working directory. The backend runs it inside the CPG's +# own scratch dir so it never lands here, but a maintainer querying a CPG by hand from the repo root +# will create one. +workspace/ diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index adee3d9..a470883 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -98,7 +98,7 @@ - _Requirements: 4.2, 4.3, 4.4, 4.5_ - _Depends: 2.1_ -- [ ] 2.3 Decide on Joern server mode, on 2.2's numbers +- [x] 2.3 Decide on Joern server mode, on 2.2's numbers - Joern offers `--server` with `--server-host`, `--server-port` and basic auth: one warm JVM with the CPG loaded once, and queries in milliseconds after that. It would remove the three-to-four JVM starts (~100s) that remain per scan now that 1.4 has batched the queries. @@ -115,6 +115,13 @@ so as a port listening on a contributor's laptop. - Observable: a recorded decision that cites 2.2's build-time-to-query-time split. A preference does not close this task; a ratio does. + - **Decided: no.** On libpng (89k lines of C) the query phase splits taint 243.3s / config 10.7s over two + invocations, and fixed overhead measured on that CPG is ~13s per invocation. So ~26s of 250s is JVM + startup and ~224s is CPGQL evaluation: a warm server removes 6% of a 446s scan and none of the part that + costs. Recorded in `benchmarks/measurements/2026-09-09-repo-libpng-envelope.json`. + - The small repository pointed the other way -- vampi's 34.3s over three kinds is roughly three startups + and almost no evaluation -- and the ratio inverts with size. That inversion is precisely why this task + was sequenced after a real repository instead of being decided on a 520-line file. - _Requirements: 4.2_ - _Depends: 2.2_ @@ -178,6 +185,30 @@ - _Requirements: 8.1, 8.2_ - _Depends: 2.6_ +- [ ] 2.9 One defect is one finding, however many regions reach it + - libpng's scan produced 26 entailed findings that are all the identical site, + `contrib/gregbook/wpng.c:335:main`. Many regions reach one shared sink and each emits its own finding; + nothing dedupes by site. A contributor would be shown the same defect twenty-six times. + - Invisible at vampi scale, where regions rarely share a sink, and it appeared the moment task 2.4 let a + region's question follow the call graph. + - Dedupe on the reported site, keeping the strongest rung and recording how many regions reached it -- + "reached from 26 entry points" is useful information, twenty-six rows are not. + - Observable: libpng's entailed count reflects distinct sites; the artifact records both numbers. + - _Requirements: 5.1, 8.1_ + - _Depends: 2.4_ + +- [ ] 2.10 Do not spend the scan on example code + - 22 of libpng's 30 highest-ranked regions are under `contrib/` -- the sample programs shipped with the + library rather than the library. The budget and the 243s taint query go there first. + - The rank already comes from the entry-point mapper; what is missing is that a path can be evidence too. + Vendored, generated, example and test trees are not what a contributor is changing. This is also the + cheapest available reduction of the 243s. + - Do not hardcode `contrib/`: derive it, record what was excluded and why, and measure the scan with and + without so the exclusion is a number rather than a preference. + - Observable: a recorded before/after on libpng -- regions, query seconds, and entailed sites. + - _Requirements: 4.2, 8.1_ + - _Depends: 2.2_ + ## Group 3 — Ship it - [ ] 3.1 Contributor output diff --git a/benchmarks/measurements/2026-09-09-repo-libpng-envelope.json b/benchmarks/measurements/2026-09-09-repo-libpng-envelope.json new file mode 100644 index 0000000..867e820 --- /dev/null +++ b/benchmarks/measurements/2026-09-09-repo-libpng-envelope.json @@ -0,0 +1,113 @@ +{ + "caveat": "The CVE is out of scope by construction, so this measures the envelope and says nothing about detection accuracy on C. The 26 entailed findings are one site, and it is in example code; no claim is made that they are correct.", + "completed": true, + "config": "shipped defaults", + "degradations": [ + { + "reason": "budget_exhausted", + "regions_unasked": 32, + "stage": "model" + }, + { + "reason": "hunter_model_unavailable", + "stage": "map" + } + ], + "exit_code": 0, + "findings_by_rung": { + "execution_confirmed": 0, + "model_corroborated": 0, + "model_entailed": 26, + "suspicion": 172 + }, + "findings_total": 198, + "fixed_invocation_overhead_seconds": 13.3, + "generated": "2026-09-09", + "known_outcomes": [ + { + "family": "memory_safety", + "id": "CVE-2025-66293", + "location": "pngread.c:png_image_read_composite:3279", + "note": "this engine arbitrates no family that decides this weakness; 'not found' would be a category error, not a miss", + "result": "out_of_scope" + } + ], + "mode": "standard", + "model": { + "arbitrate_seconds": 121.98, + "build_seconds": 16.08, + "by_rung": { + "execution_confirmed": 0, + "model_corroborated": 0, + "model_entailed": 26, + "suspicion": 9 + }, + "cost_usd": 0.0122, + "model_calls": 105, + "query_seconds": 250.0, + "regions_scanned": 121, + "regions_unasked": 32, + "regions_unjudged": 0, + "seconds": 388.06, + "unjudged_sample": [] + }, + "new_defects": [ + { + "detail": "All 26 entailed findings are the identical site, contrib/gregbook/wpng.c:335:main. Many regions reach one shared sink and each emits its own finding; nothing dedupes by site. A contributor would be shown the same defect 26 times. Invisible at vampi scale, where regions rarely share a sink.", + "id": "one-sink-many-regions-many-findings" + }, + { + "detail": "22 of the 30 highest-ranked regions are under contrib/ -- the example programs shipped with libpng, not the library. The budget and the 243s taint query are being spent on sample code.", + "id": "example-code-outranks-the-library" + } + ], + "peak_rss_mb": 911.1, + "per_kind_query_seconds": { + "config": 10.7, + "dominance": "absent: C has no handlers, so no access_control family", + "taint": 243.3 + }, + "reading": "89,082 lines of C across 93 files: CPG built in 16s at 911MB peak under a 2GB heap cap, whole scan 446s, $0.0122, 121 regions all arbitrated. Memory is not the constraint a fleet runner would hit -- a 2-vCPU/7GB runner has room for this and the rest of the job. Time is, and it is not the build: 16s to build against 250s to query.", + "repo": { + "commit": "cf53e54ccf631d8de68a30ebb3b8eeed1d6f1a08", + "known": [ + { + "family": "memory_safety", + "id": "CVE-2025-66293", + "in_scope": false, + "location": "pngread.c:png_image_read_composite:3279", + "weakness": "out-of-bounds read: an unclamped composited component indexes the sRGB conversion table" + } + ], + "language": "c", + "license": "libpng-2.0", + "measures": [ + "envelope" + ], + "name": "libpng", + "url": "https://github.com/glennrp/libpng.git" + }, + "scaling_shape": { + "answer": "No, and it is also not the pressure. The build parallelises well and costs 16s; the query does not parallelise by sharding and costs 250s. Sharding a repository across a fleet therefore splits the cheap phase, and after task 2.4 we know a shard boundary hides exactly the cross-module flows that matter. Scoping the CPG to a contributor's DIFF cuts the expensive phase instead, and is what this spec is for. Caching the built CPG as an artifact saves 16s of 446s and is barely worth it.", + "measured_ceiling": "911MB for 89k lines. Linear extrapolation puts a million-line repository near 5GB, which WOULD exceed a standard runner -- so the ceiling is real, just much further out than libpng. Locating it needs a genuinely large checkout, not this one.", + "question": "CI/CD runners are a horizontal fleet, not a vertically scaled box. Is one whole-repository CPG the only way?" + }, + "scan_seconds": 446.25, + "server_mode_decision": { + "evidence": "The query phase splits taint 243.3s / config 10.7s over two invocations. Fixed overhead measured separately on this CPG is ~13s per invocation, so ~26s of the 250s is JVM startup and ~224s is CPGQL evaluation. A warm server removes the 26s: 6% of a 446s scan, and nothing at all of the part that costs.", + "verdict": "no", + "why_the_small_repository_pointed_the_other_way": "On vampi the query phase was 34.3s over three kinds, which is roughly three startups and almost no evaluation -- startup-dominated, and it made server mode look worth having. The ratio inverts with repository size. That inversion is why task 2.3 was sequenced after a real repository rather than decided on a 520-line file." + }, + "spec": "contributor-scan 2.2/2.3 (Req 4.1-4.5) -- the envelope checkout", + "stages": { + "completed": [ + "static", + "map" + ], + "requested": [ + "static", + "map" + ], + "skipped": [] + } +} diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 3b0c1ee..5f481d9 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -890,6 +890,7 @@ def _model_payload(result: object, *, unjudged_paths: tuple[str, ...] = ()) -> d "seconds": getattr(result, "seconds", 0.0), "build_seconds": getattr(result, "build_seconds", 0.0), "query_seconds": getattr(result, "query_seconds", 0.0), + "query_seconds_by_kind": dict(getattr(result, "query_seconds_by_kind", {}) or {}), "arbitrate_seconds": getattr(result, "arbitrate_seconds", 0.0), } diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index fafd284..5c8ac25 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -83,6 +83,9 @@ class ModelScanResult: # a single figure cannot answer any of them. build_seconds: float = 0.0 query_seconds: float = 0.0 + # Per query kind, because the total cannot say whether the cost is JVM startup (which a warm server + # would remove) or CPGQL evaluation (which it would not). That is the whole of the server-mode decision. + query_seconds_by_kind: Mapping[str, float] = field(default_factory=dict) degradations: tuple[Mapping[str, object], ...] = () @property @@ -141,8 +144,10 @@ def scan_repository( # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while # the queries themselves are milliseconds once the CPG is loaded. rows_by_id: dict[str, list[object]] = {} + per_kind: dict[str, float] = {} query_started = time.monotonic() for kind, requests in _grouped(work).items(): + kind_started = time.monotonic() batch = getattr(cpg, "run_batch", None) if callable(batch): rows_by_id.update(batch(kind, requests) or {}) @@ -150,6 +155,7 @@ def scan_repository( for rid, params in requests.items(): answered = cpg.run(kind, params) rows_by_id[rid] = list(answered) if isinstance(answered, list) else [] + per_kind[kind] = round(time.monotonic() - kind_started, 2) query_seconds = round(time.monotonic() - query_started, 2) # Phase 3: arbitrate from the rows already in hand. The arbiters are unchanged: each is handed a @@ -212,6 +218,7 @@ def scan_repository( seconds=round(time.monotonic() - started, 2), build_seconds=build_seconds, query_seconds=query_seconds, + query_seconds_by_kind=per_kind, degradations=tuple(degradations), ) diff --git a/tests/test_model_scan.py b/tests/test_model_scan.py index 564bc1e..6ae1c7e 100644 --- a/tests/test_model_scan.py +++ b/tests/test_model_scan.py @@ -309,3 +309,21 @@ def test_the_call_depth_is_bounded() -> None: from openultrasast.model.scan import ENTRY_POINT_CALL_DEPTH assert 0 < ENTRY_POINT_CALL_DEPTH <= 5 + + +def test_the_scan_reports_query_time_per_kind() -> None: + """contributor-scan 2.3: which query costs the time is the whole server-mode decision. + + On libpng the phase split taint 243.3s / config 10.7s over two invocations, against ~13s of fixed + startup each. A single total cannot tell startup from evaluation, and the two lead to opposite answers. + """ + from openultrasast.model.scan import scan_repository + + regions = ( + _region(path="a.py", families=("injection",)), + _region(path="b.py", families=("config_secrets",)), + ) + result = scan_repository(Path("/repo"), regions, backend=_Backend(), client=None, model="") + + assert set(result.query_seconds_by_kind) == {"taint", "config"} + assert sum(result.query_seconds_by_kind.values()) <= result.query_seconds + 0.05