From b0500ecfa0aa866704145d2793dac25ae1cce89e Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:19:59 +0200 Subject: [PATCH] measure(contributor-scan): first ontology coverage run, and it found a family we do not have NodeGoat models 18 of 175 distinct call names, covering 12 percent of 1,185 calls. That share is not a score: most of what a program calls is not security-relevant, and a high number would mean the tables were too loose. It is a baseline to move deliberately. The top product candidate is res.render, 23 calls with five in a method that reads untrusted input. Task 14.3 reasoned its way to that same gap by hand and declined to model it as a flow sink; the instrument found it in one run, which is the validation that matters. It also surfaced Math.random in the user data-access module, and weak randomness is a weakness class this ontology has no family for at all -- so no family census could have found it, because the family list is itself the blind spot. That is the kind of gap only a coverage instrument finds. It had to be taught what is not an API. The first run ranked _tmp_3.push and __ecma.Array.factory first, which is how the frontend lowers a destructuring and an array literal, and a Cypress suite and a Gruntfile held four of the next five. Both are separated now, because noise at the top of a worklist is how a worklist stops being read. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/contributor-scan/tasks.md | 2 + ...2026-09-21-ontology-coverage-nodegoat.json | 45 +++++++++++++++++++ benchmarks/push/ontology_gap.py | 17 ++++++- src/openultrasast/cpg/queries/vocabulary.sc | 10 ++++- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 benchmarks/measurements/2026-09-21-ontology-coverage-nodegoat.json diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 0d2c798..8b85892 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -1225,6 +1225,8 @@ the ontology describes libraries, frameworks and language constructs, never repo - **It reports candidates, not entries.** A candidate becomes ontology only when someone resolves it to a library API, states the weakness class and records provenance. Nothing may be added because a subject called it. - Three defects this would have named in one run rather than three investigations: `needle.get` absent while the family found the redirect two files away, `find` absent so the documented object-reference defect raised no obligation, and `maybe_unserialize` absent so the only deserialization site in 1,164 PHP files was invisible. - Done when the instrument reports a ranked gap list for each measured subject, every entry carries how often it is called, and adding nothing to the ontology changes no result. + - _First run 2026-09-21 on NodeGoat:_ 18 of 175 distinct call names modelled, covering 12% of 1,185 calls. The top product candidate is `res.render`, 23 calls with five in a method reading untrusted input — the same gap task 14.3 reasoned its way to by hand, found here in one run, which is the validation that matters. It also surfaced `Math.random` in the user data-access module: weak randomness is a weakness class this ontology has no family for, so no census could have found it, because the family list is itself the blind spot. + - _It had to be taught what is not an API._ The first run ranked `_tmp_3.push` and `__ecma.Array.factory` first, which is how the frontend lowers a destructuring and an array literal, and a Cypress suite and a Gruntfile held four of the next five. Both are now separated, because noise at the top of a worklist is how a worklist stops being read. Evidence: `benchmarks/measurements/2026-09-21-ontology-coverage-nodegoat.json`. - _Boundary: shipped semantic facts, evaluation harness_ - _Requirements: 6.2, 8.1_ - [ ] 7.2 Key sanitizers to the weakness they actually prevent diff --git a/benchmarks/measurements/2026-09-21-ontology-coverage-nodegoat.json b/benchmarks/measurements/2026-09-21-ontology-coverage-nodegoat.json new file mode 100644 index 0000000..c7d0c3d --- /dev/null +++ b/benchmarks/measurements/2026-09-21-ontology-coverage-nodegoat.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "experiment": "ontology-coverage-first-run-v1", + "date": "2026-09-21", + "task": "contributor-scan 7.1", + "claim_scope": "What one subject calls that the ontology does not model. A candidate list, not an ontology.", + "subject": "nodegoat", + "language": "javascript", + "coverage": { + "distinct_call_names": 175, + "calls_total": 1185, + "modelled_names": 18, + "modelled_calls": 143, + "modelled_call_share": 0.121 + }, + "ontology_size": { + "discharger": 40, + "operation": 36, + "sanitizer": 10, + "setting": 10, + "sink:injection": 5, + "sink:output_encoding": 12, + "sink:path": 6, + "sink:prototype": 7, + "sink:untrusted_destination": 15, + "source": 11 + }, + "reading": "18 of 175 distinct call names are modelled, covering 12% of the calls. That number is not a score: most of what any program calls is not security-relevant, and a high share would mean the tables were too loose. It is a baseline to move deliberately.", + "leads_it_found": { + "res.render": "23 calls, five in a method that reads untrusted input, and the highest-ranked product candidate. Task 14.3 reasoned its way to this same gap by hand and declined to model it as a flow sink because template escaping is a property of the engine's configuration. The instrument found it in one run, which is the validation that matters.", + "Math.random": "7 calls in app/data/user-dao.js. Weak randomness is a weakness class this ontology has no family for at all, so no census could have surfaced it: the family list itself is the blind spot. This is the kind of gap only a coverage instrument finds.", + "parseInt": "9 calls, one near a source. A coercion, and the JavaScript sanitizer list has no coercion entry while Python's has five.", + "app.post and app.use": "route registration, near untrusted input, and the place middleware guards are attached", + "db.collection": "the accessor whose result carries the operations the access-control family models" + }, + "noise_it_had_to_be_taught_to_drop": { + "synthesised_names": "`_tmp_3.push` and `__ecma.Array.factory` were the two most frequent candidates in the first run. They are how the frontend lowers a destructuring and an array literal, not names the code calls.", + "tests_and_build_scripts": "A Cypress suite and a Gruntfile held four of the five most frequent remaining candidates. Ranked together with product code they would have buried the two real leads." + }, + "limits": [ + "The `file` and `line` on a candidate are its first occurrence, which misleads when one name is called from several files; the distinct-spelling count is reported beside it.", + "One subject, one language. The instrument says nothing about whether a candidate is security-relevant; that judgement is the point of requiring provenance before an entry.", + "Coverage share counts calls, not findings. A name modelled but never reached contributes to the share and nothing else." + ] +} diff --git a/benchmarks/push/ontology_gap.py b/benchmarks/push/ontology_gap.py index e903394..c3ee3d6 100644 --- a/benchmarks/push/ontology_gap.py +++ b/benchmarks/push/ontology_gap.py @@ -21,9 +21,14 @@ from typing import Any from openultrasast.cpg.backend import JoernBackend from openultrasast.model.contracts import ExecutionBudget +from openultrasast.model.layout import is_test_path from openultrasast.model.specs import config_specs, dominance_specs, taint_specs from openultrasast.preprocess import enumerate_source_files +# Build tooling a repository runs but does not ship. Named here rather than guessed from a path, because a +# file called Gruntfile is a build script wherever it sits. +_BUILD_SCRIPTS = frozenset({"Gruntfile.js", "gulpfile.js", "webpack.config.js", "rollup.config.js", "karma.conf.js"}) + def modelled_tokens(language: str) -> dict[str, tuple[str, ...]]: """Every token any fact table names for this language, by the role it plays. @@ -117,7 +122,17 @@ def main() -> int: } record["modelled_seen"] = [{k: r[k] for k in ("name", "count") if k in r} for r in known[: args.top]] wanted = ("name", "spelling", "count", "nearSource", "file", "line") - record["candidates"] = [{k: r[k] for k in wanted if k in r} for r in unknown[: args.top]] + + # A test helper and a build script are not product API. The layout facts already know which paths those + # are, and on the first measured subject a Gruntfile and a Cypress suite held four of the five most + # frequent candidates -- ranked first, they would have buried the two real leads beneath them. + def product(row: dict[str, Any]) -> bool: + where = str(row.get("file") or "") + return bool(where) and not is_test_path(where) and Path(where).name not in _BUILD_SCRIPTS + + ranked = [{k: r[k] for k in wanted if k in r} for r in unknown] + record["candidates"] = [r for r in ranked if product(r)][: args.top] + record["candidates_in_test_or_build"] = [r for r in ranked if not product(r)][:20] record["candidates_near_a_source"] = [r for r in record["candidates"] if int(r.get("nearSource", 0)) > 0][:20] save() cleanup = getattr(graph, "cleanup", None) diff --git a/src/openultrasast/cpg/queries/vocabulary.sc b/src/openultrasast/cpg/queries/vocabulary.sc index 3285361..7967bc8 100644 --- a/src/openultrasast/cpg/queries/vocabulary.sc +++ b/src/openultrasast/cpg/queries/vocabulary.sc @@ -36,7 +36,14 @@ head.trim.takeRight(120) } - val calls = cpg.call.filterNot(_.name.startsWith(" c.name.startsWith(" items.exists(_.modelledHere), "nearSource" -> items.count(_.near), "file" -> first.file, + "spellings" -> items.map(_.spelling).distinct.size, "line" -> first.line ) }