mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
9ed0e45c5f
99 source modules -> 81. 832 tests -> 646. The whole learning/ package (16 modules, ~4,100 lines) and six semantic/ modules removed, with the three gates byte-identical to 2026-09-06-gate-baseline.txt at every step. The sequence is what made it safe: vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion third (1.4). Every import break during the deletion was therefore a detachment miss in the importer and was fixed there -- no module was restored to satisfy one. Both misses were the same shape and were fixed as a class: operations.py and shapes.py each imported trailing_name from the deleted variants.py, so it moved to semantic/ir.py, its natural home, and both importers changed in one edit. Ported before deleting: the closed guard vocabulary (verbatim, nothing else holds it) into model/specs.py, which derives TaintSpec and DominanceSpec from the retained ruleset facts rather than copying them. Re-homed: families -> model/taxonomy, candidates -> model/candidates, endpoint -> model/endpoint, plus Region, labeled_family and labeled_spans. labeled_family no longer falls back to the LLM classifier -- an unlabeled pair is honestly unknown. Four deviations, each recorded in the spec notes with its reason. improve/ evolve.py and validator.py were stripped, not deleted: each mixes the mechanism lever (gone) with the deterministic rule-status loop `ousast improve` still runs, which has no LLM in it; a test now asserts the lever's absence by symbol so it cannot creep back. The learning CLI group became `model candidates`, because deleting it outright would have orphaned the retained enumerator by removing its only entry point. thinking moved from [learning] to [models]. A gap named rather than papered over: no retained sink fact routes by CWE to path, output_encoding, untrusted_destination or prototype -- the same hole the entailment ceiling diagnosed. The test asserts the seeded set and the gap set exactly, so task 4.2 must update it when it authors those sinks. The audit found exactly one genuine orphan: vectorstore.py, a bakeoff harness with no production caller and no CLI, alive only through its own test. The five named subsystems each carry an explicit decision. Manifest committed, with a test asserting the tree still matches it. Group 2 (the Joern go/no-go) has not started. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
102 lines
5.1 KiB
Python
102 lines
5.1 KiB
Python
"""model-grounded-detection task 1.1: the security vocabularies survive the modules that carried them.
|
|
|
|
The guard vocabulary lived in `semantic/variants.py`, whose optimisation loop this feature deletes. The
|
|
source/sink/sanitizer facts and the obligation facts live in `ruleset/` data and are retained, so the specs
|
|
*derive* from them rather than copying them — a copy would be a second place to update. What must be ported
|
|
verbatim is the guard vocabulary, because nothing else holds it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
from pathlib import Path
|
|
|
|
SPECS = Path("src/openultrasast/model/specs.py")
|
|
|
|
|
|
def test_the_guard_vocabulary_is_preserved_verbatim() -> None:
|
|
from openultrasast.model.specs import GUARD_KINDS
|
|
|
|
assert GUARD_KINDS == ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none")
|
|
|
|
|
|
def test_the_guard_classifier_still_recognises_each_kind() -> None:
|
|
"""The patterns classify the *fix*, first match wins — the order is part of the vocabulary."""
|
|
from openultrasast.model.specs import classify_guard_text
|
|
|
|
assert classify_guard_text('cur.execute("SELECT * FROM t WHERE id = %s", (uid,))') == "parameterized_call"
|
|
assert classify_guard_text("if not current_user.is_authenticated: abort(403)") == "auth_check"
|
|
assert classify_guard_text('if name not in ALLOWED: abort(400)') == "allowlist_test"
|
|
assert classify_guard_text("if len(buf) > 64: return") == "bounds_test"
|
|
assert classify_guard_text("if value is None: return") == "null_test"
|
|
assert classify_guard_text("page = int(raw)") == "type_change"
|
|
assert classify_guard_text("total = total + 1") == "none"
|
|
|
|
|
|
def test_the_seeded_families_and_the_named_gaps_are_both_exact() -> None:
|
|
"""What the retained fact data can seed, and what it provably cannot — Req 9.2, a gap is *named*.
|
|
|
|
`path`, `output_encoding`, `untrusted_destination` and `prototype` have no sink in `ruleset/semantic`
|
|
whose CWE routes to them, which is the same hole the entailment ceiling diagnosed ("closed literal sink
|
|
table: misses per-family sinks"). Task 4.2 authors those sinks and must update this test when it does —
|
|
the set is asserted exactly so a gap can neither appear nor close silently.
|
|
"""
|
|
from openultrasast.model.specs import dominance_specs, taint_specs
|
|
from openultrasast.model.taxonomy import load_families
|
|
|
|
taxonomy = load_families()
|
|
seeded = set()
|
|
for language in ("python", "javascript", "java", "c"):
|
|
seeded |= {spec.family for spec in taint_specs(language=language).values()}
|
|
seeded |= {spec.family for spec in dominance_specs(language=language).values()}
|
|
deferred = {"unknown", "memory"} # the declined bucket and the deferred execution tier
|
|
assert seeded == {"injection", "deserialization", "config_secrets", "access_control", "memory"}
|
|
gaps = {family.id for family in taxonomy.families} - seeded - deferred
|
|
assert gaps == {"path", "output_encoding", "untrusted_destination", "prototype"}, (
|
|
"a family lost or gained a seed without this record being updated"
|
|
)
|
|
|
|
|
|
def test_the_injection_taint_spec_carries_sources_sinks_and_sanitizers() -> None:
|
|
from openultrasast.model.specs import taint_specs
|
|
|
|
spec = taint_specs(language="python")["injection"]
|
|
assert any("request.args" in source for source in spec.sources)
|
|
assert any(sink in ("eval", "exec", "os.system") for sink in spec.sinks)
|
|
assert spec.sanitizers, "a family with no sanitizer model can never distinguish a fixed twin"
|
|
|
|
|
|
def test_the_access_control_dominance_spec_carries_operations_and_dischargers() -> None:
|
|
from openultrasast.model.specs import dominance_specs
|
|
|
|
spec = dominance_specs(language="python")["access_control"]
|
|
assert any("filter_by" in operation for operation in spec.operations)
|
|
assert spec.dischargers, "an absence family with no discharger model can only ever report suspicion"
|
|
|
|
|
|
def test_specs_import_nothing_from_the_removed_packages() -> None:
|
|
"""Req 1.2: the vocabularies must not depend on the modules this feature deletes."""
|
|
tree = ast.parse(SPECS.read_text())
|
|
imported = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.module:
|
|
imported.add(node.module)
|
|
elif isinstance(node, ast.Import):
|
|
imported.update(alias.name for alias in node.names)
|
|
forbidden = ("learning", "improve", "variants", "mechanisms", "evolve")
|
|
offending = [name for name in imported if any(token in name for token in forbidden)]
|
|
assert not offending, f"model/specs.py imports removed code: {offending}"
|
|
|
|
|
|
def test_the_specs_module_exposes_no_writer() -> None:
|
|
"""Req 7.4, the verifier boundary: the models are read-only data no LLM or optimiser can rewrite."""
|
|
tree = ast.parse(SPECS.read_text())
|
|
writers = [
|
|
node.name
|
|
for node in tree.body
|
|
if isinstance(node, ast.FunctionDef)
|
|
and any(node.name.startswith(verb) for verb in ("write", "save", "dump", "store", "append", "update", "set_"))
|
|
]
|
|
assert not writers, f"specs must be read-only, found writers: {writers}"
|
|
assert "open(" not in SPECS.read_text().replace("# ", ""), "specs must not open a file for writing"
|