marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 11:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pre-push-safety-net): census the declared families and fix a sink matched in a comment

Browse Source 
Seven vulnerability families are declared and two had been proven. The census asks each of
them, on whole checkouts of NodeGoat and VAmPI, whether it establishes anything at all, and
reports a family that answered and found nothing separately from one that was never asked.

Four establish findings: injection, access_control, config_secrets and untrusted_destination.
Output encoding, path, prototype and deserialization establish nothing on either subject. All
299 NodeGoat and 35 VAmPI questions completed, so the silences are answers.

Three of those silences are diagnosed gaps, not clean code. NodeGoat's documented SSRF calls
needle.get, which the sink vocabulary does not hold. Its XSS is server-rendered, and output
encoding declares only DOM sinks, so no question it can ask could have found it. Its two
access-control defects raise no obligation at all: measured with the real dominance query,
every modelled data operation lives in app/data/*-dao.js and none in a route handler file,
while the family scopes operations to the region's own file. Tasks 14.2 to 14.5 carry each
gap; none is fixed here.

The census also found a false positive and it is fixed. NodeGoat's only prototype report was
the word `set` inside `// set these up in case we have an error case`: the matched node is the
<operator>.assignment holding the entire 1,000-character signup handler, and sinkMatches
searched a node's whole text. It now confines the text clause to the callee and excludes
operator nodes. Fourth instance of this bug class after fgets/gets, resolve/resolveUrl and
user/users. Measured on NodeGoat's non-vendor calls, injection sinks fall 28 to 3, prototype
13 to 2 and untrusted destination 26 to 7, with no true positive lost; req.query had been an
injection sink because `query` is word-bounded inside it. Witnesses now name the sink call
rather than the assignment that contained it.

1,411 tests pass, ten skipped; the six Joern-gated query tests pass in the image, including a
new one that fails on the shape this fixed.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-19 17:39:29 +02:00
co-authored by Claude Opus 5
parent 770bae0e9e
commit 4e2619e248
6 changed files with 646 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-09-19-family-detection-census.json
+363
View File
@@ -0,0 +1,363 @@
{
"schema_version": 1,
"experiment": "family-detection-census-v1",
"date": "2026-09-19",
"harness": "benchmarks/push/family_census.py",
"claim_scope": "Detection only, on two inspected teaching repositories. No comparison, no novelty, no admission, no capability qualified. These subjects developed the analyser and cannot qualify it.",
"question": "Which of the seven declared vulnerability families establish a finding on real code, and what does each silent family mean?",
"subjects": {
"nodegoat": {
"schema_version": 1,
"experiment": "family-detection-census-v1",
"subject": "nodegoat",
"root": "/cache/nodegoat/c5cb68a7084e",
"claim_scope": "Detection only. No comparison, no novelty, no admission, no capability enabled.",
"inputs": {
"files": 44,
"bytes": 95411,
"regions": 44
},
"regions_by_family": {
"access_control": 35,
"config_secrets": 44,
"injection": 44,
"output_encoding": 44,
"path": 44,
"prototype": 44,
"untrusted_destination": 44
},
"languages": {
"javascript": 44
},
"scan": {
"seconds": 137.3,
"build_seconds": 28.66,
"query_seconds": 74.48,
"hook_table_hooks": 0,
"degradations": [
"vendor_semantics_unresolved"
],
"questions_total": 299,
"questions_completed": 299
},
"families": {
"access_control": {
"regions_offering": 35,
"questions_asked": 35,
"questions_completed": 35,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "answered, nothing found"
},
"config_secrets": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 1,
"rungs": {
"model_entailed": 1
},
"examples": [
{
"site": "server.js:78:?",
"rung": "model_entailed",
"witness": "session({\n // genid: (req) => {\n // return genuuid() // use UUIDs for session IDs\n //},\n secret: cookieSecret,\n // Both mandatory in Express v4\n saveUninit",
"reached_from": 1
}
],
"verdict": "establishes findings"
},
"injection": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 3,
"rungs": {
"model_entailed": 3
},
"examples": [
{
"site": "app/routes/contributions.js:32:<lambda>2",
"rung": "model_entailed",
"witness": "req.body.preTax -> eval(req.body.preTax) in app/routes/contributions.js (line 32, 1 steps)",
"reached_from": 1
},
{
"site": "app/routes/contributions.js:33:<lambda>2",
"rung": "model_entailed",
"witness": "req.body.afterTax -> eval(req.body.afterTax) in app/routes/contributions.js (line 33, 1 steps)",
"reached_from": 1
},
{
"site": "app/routes/contributions.js:34:<lambda>2",
"rung": "model_entailed",
"witness": "req.body.roth -> eval(req.body.roth) in app/routes/contributions.js (line 34, 1 steps)",
"reached_from": 1
}
],
"verdict": "establishes findings"
},
"output_encoding": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "answered, nothing found"
},
"path": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "answered, nothing found"
},
"prototype": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "answered, nothing found"
},
"untrusted_destination": {
"regions_offering": 44,
"questions_asked": 44,
"questions_completed": 44,
"findings": 1,
"rungs": {
"model_entailed": 1
},
"examples": [
{
"site": "app/routes/index.js:72:<lambda>1",
"rung": "model_entailed",
"witness": "req.query.url -> res.redirect(req.query.url) in app/routes/index.js (line 72, 1 steps)",
"reached_from": 1
}
],
"verdict": "establishes findings"
}
},
"summary": {
"answered, nothing found": 4,
"establishes findings": 3
}
},
"vampi": {
"schema_version": 1,
"experiment": "family-detection-census-v1",
"subject": "vampi",
"root": "/cache/vampi/f16052dce83f",
"claim_scope": "Detection only. No comparison, no novelty, no admission, no capability enabled.",
"inputs": {
"files": 11,
"bytes": 19907,
"regions": 25
},
"regions_by_family": {
"config_secrets": 25,
"deserialization": 25,
"injection": 25,
"output_encoding": 25,
"path": 25,
"untrusted_destination": 25,
"access_control": 8
},
"languages": {
"python": 25
},
"scan": {
"seconds": 85.3,
"build_seconds": 8.82,
"query_seconds": 52.26,
"hook_table_hooks": 0,
"degradations": [],
"questions_total": 35,
"questions_completed": 35
},
"families": {
"access_control": {
"regions_offering": 8,
"questions_asked": 8,
"questions_completed": 8,
"findings": 2,
"rungs": {
"model_entailed": 2
},
"examples": [
{
"site": "api_views/books.py:51:get_by_title",
"rung": "model_entailed",
"witness": "get_by_title line 51: no discharging guard, while 1 sibling handler(s) are guarded (add_new_book)",
"reached_from": 1
},
{
"site": "api_views/users.py:187:update_password",
"rung": "model_entailed",
"witness": "update_password line 187: no discharging guard, while 3 sibling handler(s) are guarded (delete_user, me, update_email)",
"reached_from": 1
}
],
"verdict": "establishes findings"
},
"config_secrets": {
"regions_offering": 25,
"questions_asked": 25,
"questions_completed": 25,
"findings": 1,
"rungs": {
"model_entailed": 1
},
"examples": [
{
"site": "app.py:17:<module>",
"rung": "model_entailed",
"witness": "vuln_app.run(host = '0.0.0.0', port = 5000, debug = True) (line 17): permissive literal '0.0.0.0'",
"reached_from": 1
}
],
"verdict": "establishes findings"
},
"deserialization": {
"regions_offering": 25,
"questions_asked": 0,
"questions_completed": 0,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "region offered, no question asked"
},
"injection": {
"regions_offering": 25,
"questions_asked": 2,
"questions_completed": 2,
"findings": 1,
"rungs": {
"model_entailed": 1
},
"examples": [
{
"site": "models/user_model.py:73:get_user",
"rung": "model_entailed",
"witness": "username -> text(user_query) in models/user_model.py (line 73, 13 steps)",
"reached_from": 1
}
],
"verdict": "establishes findings"
},
"output_encoding": {
"regions_offering": 25,
"questions_asked": 0,
"questions_completed": 0,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "region offered, no question asked"
},
"path": {
"regions_offering": 25,
"questions_asked": 0,
"questions_completed": 0,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "region offered, no question asked"
},
"untrusted_destination": {
"regions_offering": 25,
"questions_asked": 0,
"questions_completed": 0,
"findings": 0,
"rungs": {},
"examples": [],
"verdict": "region offered, no question asked"
}
},
"summary": {
"establishes findings": 3,
"region offered, no question asked": 4
}
}
},
"verdict": {
"establish_findings": [
"injection",
"access_control",
"config_secrets",
"untrusted_destination"
],
"answered_nothing_found": {
"nodegoat": [
"access_control",
"output_encoding",
"path",
"prototype"
],
"vampi": []
},
"region_offered_no_question_asked": {
"nodegoat": [],
"vampi": [
"deserialization",
"output_encoding",
"path",
"untrusted_destination"
]
},
"never_established_anywhere": [
"output_encoding",
"path",
"prototype",
"deserialization"
]
},
"false_positive_found_and_fixed": {
"family": "prototype",
"site": "app/routes/session.js:189",
"reported_witness": "req.body -> this.handleSignup = (req, res, next) => { ...",
"measured_cause": "The matched node is <operator>.assignment whose code is the entire 1,000-character arrow function. The prototype sink 'set' matched the English word in the comment '// set these up in case we have an error case'. The handler calls no set.",
"fix": "taint.sc sinkMatches confines its text clause to the callee and excludes operator nodes",
"measured_effect_on_nodegoat_non_vendor_calls": {
"injection": {
"old": 28,
"new": 3
},
"output_encoding": {
"old": 3,
"new": 0
},
"path": {
"old": 0,
"new": 0
},
"prototype": {
"old": 13,
"new": 2
},
"untrusted_destination": {
"old": 26,
"new": 7
}
},
"true_positives_lost": 0,
"note": "Every dropped match is an operator node or an enclosing call whose body merely spells the word. req.query was an injection sink under the old rule, because 'query' is word-bounded inside it. The three eval calls and the res.redirect survive, and their witnesses now name the sink call rather than the assignment that contained it."
},
"silent_families_diagnosed": {
"untrusted_destination_on_nodegoat": "NodeGoat's documented SSRF is needle.get(req.query.url + req.query.symbol) at app/routes/research.js:16. needle.get is absent from the sink vocabulary, which holds axios/fetch/got/http.get/request. The family works; its vocabulary is thin. Recall gap.",
"output_encoding_on_nodegoat": "All six declared sinks are DOM sinks (innerHTML, document.write, dangerouslySetInnerHTML, insertAdjacentHTML, outerHTML, v-html). NodeGoat's XSS is server-side, rendered into Swig templates and written with res.write, so no question the family can ask could have found it. Structural gap: server-rendered output has no sink vocabulary in JavaScript.",
"access_control_on_nodegoat": "Measured with the real dominance query: every modelled data operation in NodeGoat lives in app/data/*-dao.js or artifacts/db-reset.js, and none in a route handler file. Operations are scoped to the region's own file, and access_control regions are the handler files, so no obligation is ever raised where the question is asked. In the DAO files every operation is equally unguarded, so the sibling-contradiction rung has nothing to contradict. The two documented defects are also outside the model's expressible claims: an IDOR that reads userId from req.params on an authenticated route (app/routes/allocations.js:17), and a commented-out isAdmin in the route table (app/routes/index.js:58). Structural gap: the model asks about one file, and this application splits route from query.",
"four_families_on_vampi": "deserialization, output_encoding, path and untrusted_destination asked nothing because VAmPI's eleven Python files contain no sink for any of them; the only match of any such token is os.path.join in config.py. Correct silence, not a gap.",
"prototype": "Never established anywhere. Its only report was the false positive above. VAmPI is Python and the family is JavaScript-only, so it has been asked on one repository."
},
"limits": [
"Two repositories, both used to develop the analyser. Nothing here qualifies a capability.",
"Findings are counted at the rung the scan reached, all model_entailed; no execution confirmation.",
"A family silent on two repositories is untested, not absent."
]
}
benchmarks/push/family_census.py
+139
View File
@@ -0,0 +1,139 @@
"""Which declared vulnerability families actually establish findings on real code?
Seven families are declared. Two are proven end to end: injection on NodeGoat and VAmPI, and access
control on VAmPI once its change context could complete. The other five have never been checked for
whether they produce a witness on anything, and a family that is declared but silent is advertised
coverage the tool does not have.
This runs the production discovery and scan over a whole checkout and reports, per family, how many
regions were asked, how many questions completed, and what was found with the witness and rung that
established it. It makes no novelty or admission claim: there is no comparison here, only detection.
A family reporting nothing is recorded as nothing. That is the point of asking.
"""
from __future__ import annotations
import argparse
import collections
import json
import time
from pathlib import Path
from typing import Any
from openultrasast.cpg.backend import JoernBackend
from openultrasast.mapping import analyze_entry_points, php_hook_callbacks
from openultrasast.model.contracts import ExecutionBudget
from openultrasast.model.regions import regions_for
from openultrasast.model.scan import ScanBudget, scan_repository
from openultrasast.model.shipped import declared_sources
from openultrasast.preprocess import build_file_target, enumerate_source_files
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--root", type=Path, required=True)
parser.add_argument("--name", required=True)
parser.add_argument("--deadline", type=float, default=1800.0)
parser.add_argument("--max-regions", type=int, default=500)
args = parser.parse_args()
args.out.mkdir(parents=True, exist_ok=True)
record: dict[str, Any] = {
"schema_version": 1,
"experiment": "family-detection-census-v1",
"subject": args.name,
"root": str(args.root),
"claim_scope": "Detection only. No comparison, no novelty, no admission, no capability enabled.",
}
target = args.out / f"{args.name}.json"
def save() -> None:
target.write_text(json.dumps(record, indent=2, default=str) + "\n")
files = enumerate_source_files(args.root)
targets = [build_file_target(args.root, path) for path in files]
opened = sum(path.stat().st_size for path in files if path.is_file())
regions = regions_for(analyze_entry_points(args.root, targets), targets, shipped=declared_sources(args.root))
record["inputs"] = {"files": len(files), "bytes": opened, "regions": len(regions)}
record["regions_by_family"] = dict(collections.Counter(f for r in regions for f in r.families))
record["languages"] = dict(collections.Counter(r.language for r in regions))
save()
if not files or not opened:
record["error"] = "no readable source"
save()
return 2
backend = JoernBackend(execution_budget=ExecutionBudget(time.monotonic() + args.deadline, 5.0))
hooks = php_hook_callbacks(args.root, targets) if any(r.language == "php" for r in regions) else {}
started = time.monotonic()
result = scan_repository(
args.root,
regions[: args.max_regions],
backend=backend,
budget=ScanBudget(max_model_calls=0, max_regions=args.max_regions, order_by_evidence=True),
execution_budget=ExecutionBudget(time.monotonic() + args.deadline, 5.0),
ranking_mode="evidence",
unit="repository",
population_complete=True,
)
record["scan"] = {
"seconds": round(time.monotonic() - started, 1),
"build_seconds": result.build_seconds,
"query_seconds": result.query_seconds,
"hook_table_hooks": len(hooks),
"degradations": [str(d.get("reason")) for d in result.degradations],
"questions_total": len(result.question_outcomes),
"questions_completed": sum(1 for q in result.question_outcomes if q.status == "completed"),
}
asked = collections.Counter(q.identity.family for q in result.question_outcomes)
completed = collections.Counter(q.identity.family for q in result.question_outcomes if q.status == "completed")
found: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
for finding in result.findings:
found[finding.family].append(
{
"site": finding.site,
"rung": finding.rung.value,
"witness": (finding.witness or "")[:200],
"reached_from": finding.reached_from,
}
)
# Families with regions but no question must appear, or the census hides its own biggest finding: on
# VAmPI four declared families were offered 25 regions each and asked nothing, and building this list
# from the questions alone made them invisible rather than negative.
offered = record["regions_by_family"]
families = sorted(set(offered) | set(asked) | set(found))
record["families"] = {
family: {
"regions_offering": offered.get(family, 0),
"questions_asked": asked.get(family, 0),
"questions_completed": completed.get(family, 0),
"findings": len(found.get(family, [])),
"rungs": dict(collections.Counter(item["rung"] for item in found.get(family, []))),
"examples": found.get(family, [])[:3],
# The distinction that matters: a family that answered and found nothing is a different
# statement from one that could not answer, and neither is "clean".
"verdict": (
"region offered, no question asked"
if not asked.get(family)
else "asked but no question completed"
if not completed.get(family)
else "answered, nothing found"
if not found.get(family)
else "establishes findings"
),
}
for family in families
}
record["summary"] = dict(collections.Counter(entry["verdict"] for entry in record["families"].values()))
save()
cleanup = getattr(result, "cleanup", None)
if callable(cleanup):
cleanup()
print(json.dumps({"subject": args.name, "summary": record["summary"]}), flush=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())