mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
9ed0e45c5f
99 source modules -> 81. 832 tests -> 646. The whole learning/ package (16 modules, ~4,100 lines) and six semantic/ modules removed, with the three gates byte-identical to 2026-09-06-gate-baseline.txt at every step. The sequence is what made it safe: vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion third (1.4). Every import break during the deletion was therefore a detachment miss in the importer and was fixed there -- no module was restored to satisfy one. Both misses were the same shape and were fixed as a class: operations.py and shapes.py each imported trailing_name from the deleted variants.py, so it moved to semantic/ir.py, its natural home, and both importers changed in one edit. Ported before deleting: the closed guard vocabulary (verbatim, nothing else holds it) into model/specs.py, which derives TaintSpec and DominanceSpec from the retained ruleset facts rather than copying them. Re-homed: families -> model/taxonomy, candidates -> model/candidates, endpoint -> model/endpoint, plus Region, labeled_family and labeled_spans. labeled_family no longer falls back to the LLM classifier -- an unlabeled pair is honestly unknown. Four deviations, each recorded in the spec notes with its reason. improve/ evolve.py and validator.py were stripped, not deleted: each mixes the mechanism lever (gone) with the deterministic rule-status loop `ousast improve` still runs, which has no LLM in it; a test now asserts the lever's absence by symbol so it cannot creep back. The learning CLI group became `model candidates`, because deleting it outright would have orphaned the retained enumerator by removing its only entry point. thinking moved from [learning] to [models]. A gap named rather than papered over: no retained sink fact routes by CWE to path, output_encoding, untrusted_destination or prototype -- the same hole the entailment ceiling diagnosed. The test asserts the seeded set and the gap set exactly, so task 4.2 must update it when it authors those sinks. The audit found exactly one genuine orphan: vectorstore.py, a bakeoff harness with no production caller and no CLI, alive only through its own test. The five named subsystems each carry an explicit decision. Manifest committed, with a test asserting the tree still matches it. Group 2 (the Joern go/no-go) has not started. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
80 lines
3.7 KiB
Python
80 lines
3.7 KiB
Python
"""What every optional dependency's absence looks like, and that the gates never move (task 5.1, Req 11.4, 11.5).
|
|
|
|
The core install has no dependencies. Every capability beyond it is optional, and the rule for all of them is the
|
|
same: when it is missing the run says so by name and finishes, rather than crashing or — worse — quietly producing
|
|
a smaller number that reads like a result. Each absence is exercised once here, with the reason it records.
|
|
|
|
The gate baseline is the other half. Three gates are the only place in this project where a detector change is
|
|
allowed to be invisible; if one of them moves, something that was never supposed to change did.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
BASELINE = Path("benchmarks/measurements/2026-09-06-gate-baseline.txt")
|
|
MARKER = "# --- output begins ---\n"
|
|
|
|
|
|
def test_the_three_gate_outputs_match_the_committed_pre_feature_baseline() -> None:
|
|
"""Req 11.5. The baseline was captured at `71efc9c`, before the first line of this feature."""
|
|
assert BASELINE.is_file()
|
|
expected = BASELINE.read_text().split(MARKER, 1)[1]
|
|
produced = "".join(
|
|
subprocess.run( # noqa: S603 — this repository's own modules, no shell
|
|
[sys.executable, "-m", f"openultrasast.{module}"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
).stdout
|
|
for module in ("gate", "map_gate", "pair_gate")
|
|
)
|
|
assert produced == expected, "a gate moved; diff this against benchmarks/measurements/2026-09-06-gate-baseline.txt"
|
|
|
|
|
|
def test_the_baseline_records_where_it_came_from() -> None:
|
|
head = BASELINE.read_text().split(MARKER, 1)[0]
|
|
assert "71efc9c" in head and "openultrasast.pair_gate" in head
|
|
|
|
|
|
# --- one optional dependency at a time ---------------------------------------
|
|
|
|
|
|
def test_without_the_network_a_pointer_pair_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from openultrasast.pairs import evaluate_catalog, load_pair_catalog, select_slice
|
|
|
|
monkeypatch.delenv("OPENULTRASAST_PAIRS_NETWORK", raising=False)
|
|
pointers = [case for case in select_slice(load_pair_catalog(), "vibe-py") if not case.vendored][:2]
|
|
assert pointers, "the vibe-py slice carries pointer rows; without them this proves nothing"
|
|
result = evaluate_catalog(pointers, pointers=False)
|
|
assert {item["reason"] for item in result.degradations} == {"pointer_pair_skipped"}
|
|
assert result.outcomes == ()
|
|
|
|
|
|
def test_without_the_semantic_extra_an_unsupported_language_is_named_not_guessed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from openultrasast.semantic import extra as extra_module
|
|
from openultrasast.semantic.ir import parse_file
|
|
|
|
monkeypatch.setattr(extra_module, "has_semantic_extra", lambda: False)
|
|
ir = parse_file("app.go", "package main\n\nfunc main() {}\n", "go")
|
|
assert ir.reason == "language_unsupported" and ir.parse_ok is False
|
|
|
|
|
|
def test_without_a_sandbox_the_regress_stage_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A claim that could not be proved because nothing could run it is not the same as a claim that failed."""
|
|
import json
|
|
|
|
from openultrasast.cli import main
|
|
|
|
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
|
(tmp_path / "app.py").write_text("import os\n\n\ndef run(cmd):\n os.system(cmd)\n")
|
|
assert main(["scan", str(tmp_path), "--mode", "deep"]) in {0, 1}
|
|
runs = sorted((tmp_path / ".openultrasast" / "runs").iterdir())
|
|
manifest = json.loads((runs[-1] / "manifest.json").read_text())
|
|
reasons = {item["reason"] for item in manifest.get("degradations", [])}
|
|
assert "sandbox_unavailable" in reasons
|