mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 03:59:28 +00:00
feat(model-grounded-detection): group 1, subtract the noise architecture
99 source modules -> 81. 832 tests -> 646. The whole learning/ package (16 modules, ~4,100 lines) and six semantic/ modules removed, with the three gates byte-identical to 2026-09-06-gate-baseline.txt at every step. The sequence is what made it safe: vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion third (1.4). Every import break during the deletion was therefore a detachment miss in the importer and was fixed there -- no module was restored to satisfy one. Both misses were the same shape and were fixed as a class: operations.py and shapes.py each imported trailing_name from the deleted variants.py, so it moved to semantic/ir.py, its natural home, and both importers changed in one edit. Ported before deleting: the closed guard vocabulary (verbatim, nothing else holds it) into model/specs.py, which derives TaintSpec and DominanceSpec from the retained ruleset facts rather than copying them. Re-homed: families -> model/taxonomy, candidates -> model/candidates, endpoint -> model/endpoint, plus Region, labeled_family and labeled_spans. labeled_family no longer falls back to the LLM classifier -- an unlabeled pair is honestly unknown. Four deviations, each recorded in the spec notes with its reason. improve/ evolve.py and validator.py were stripped, not deleted: each mixes the mechanism lever (gone) with the deterministic rule-status loop `ousast improve` still runs, which has no LLM in it; a test now asserts the lever's absence by symbol so it cannot creep back. The learning CLI group became `model candidates`, because deleting it outright would have orphaned the retained enumerator by removing its only entry point. thinking moved from [learning] to [models]. A gap named rather than papered over: no retained sink fact routes by CWE to path, output_encoding, untrusted_destination or prototype -- the same hole the entailment ceiling diagnosed. The test asserts the seeded set and the gap set exactly, so task 4.2 must update it when it authors those sinks. The audit found exactly one genuine orphan: vectorstore.py, a bakeoff harness with no production caller and no CLI, alive only through its own test. The five named subsystems each carry an explicit decision. Manifest committed, with a test asserting the tree still matches it. Group 2 (the Joern go/no-go) has not started. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
fc4dab97fb
commit
9ed0e45c5f
@@ -16,7 +16,7 @@ import pytest
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_obligation_operations import APP # noqa: E402
|
||||
|
||||
from openultrasast.learning.families import load_families # noqa: E402
|
||||
from openultrasast.model.taxonomy import load_families # noqa: E402
|
||||
from openultrasast.semantic.extra import has_semantic_extra # noqa: E402
|
||||
|
||||
INJECTION = (
|
||||
@@ -33,7 +33,7 @@ INJECTION = (
|
||||
|
||||
|
||||
def _region(path: str = "app.py", function: str | None = "run"): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.detectors import Region
|
||||
from openultrasast.model.candidates import Region
|
||||
|
||||
return Region(path=path, function=function)
|
||||
|
||||
@@ -44,7 +44,7 @@ def _write(tmp_path: Path, name: str, text: str) -> Path:
|
||||
|
||||
|
||||
def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", INJECTION)
|
||||
result = enumerate_candidates(root, _region(), "injection", taxonomy=load_families())
|
||||
@@ -62,7 +62,7 @@ def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path
|
||||
|
||||
|
||||
def test_the_family_decides_which_kinds_are_candidates() -> None:
|
||||
from openultrasast.learning.candidates import FAMILY_SHAPE
|
||||
from openultrasast.model.candidates import FAMILY_SHAPE
|
||||
|
||||
taxonomy = load_families()
|
||||
assert set(FAMILY_SHAPE) == {family.id for family in taxonomy.families}, "every family says what carries its bug"
|
||||
@@ -76,7 +76,7 @@ def test_the_family_decides_which_kinds_are_candidates() -> None:
|
||||
@pytest.mark.skipif(not has_semantic_extra(), reason="the guards come from the parsed IR")
|
||||
def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_path: Path) -> None:
|
||||
"""An absence bug has no sink. Its candidate is the operation, and what the model needs is what guards it."""
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
guarded = enumerate_candidates(root, _region(function="guarded"), "access_control", taxonomy=load_families())
|
||||
@@ -95,7 +95,7 @@ def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_p
|
||||
|
||||
|
||||
def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", "def broken(:\n pass\n")
|
||||
result = enumerate_candidates(root, _region(function="broken"), "injection", taxonomy=load_families())
|
||||
@@ -103,7 +103,7 @@ def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: P
|
||||
|
||||
|
||||
def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", "def empty():\n return 1\n")
|
||||
result = enumerate_candidates(root, _region(function="empty"), "injection", taxonomy=load_families())
|
||||
@@ -111,7 +111,7 @@ def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Pa
|
||||
|
||||
|
||||
def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates
|
||||
from openultrasast.model.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates
|
||||
|
||||
# one call per line, so the arithmetic is the bound and not the fixture
|
||||
body = "\n".join(f" sink_{index}(value)" for index in range(MAX_CANDIDATES_PER_CALL + 5))
|
||||
@@ -123,7 +123,7 @@ def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) ->
|
||||
|
||||
def test_enumerating_twice_returns_an_identical_sequence(tmp_path: Path) -> None:
|
||||
"""The batches, and therefore the sequence of model calls, are determined by the input alone (Req 2.4)."""
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
taxonomy = load_families()
|
||||
@@ -139,7 +139,7 @@ def _order(result): # type: ignore[no-untyped-def]
|
||||
|
||||
|
||||
def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", APP)
|
||||
whole = enumerate_candidates(root, _region(function=None), "injection", taxonomy=load_families())
|
||||
@@ -150,7 +150,7 @@ def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) -
|
||||
|
||||
@pytest.mark.parametrize("family", ["injection", "path", "deserialization", "untrusted_destination", "config_secrets"])
|
||||
def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: Path, family: str) -> None:
|
||||
from openultrasast.learning.candidates import enumerate_candidates
|
||||
from openultrasast.model.candidates import enumerate_candidates
|
||||
|
||||
root = _write(tmp_path, "app.py", INJECTION)
|
||||
result = enumerate_candidates(root, _region(), family, taxonomy=load_families())
|
||||
@@ -164,7 +164,7 @@ def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path:
|
||||
|
||||
def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_path: Path) -> None:
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.learning.candidates import ceiling
|
||||
from openultrasast.model.candidates import ceiling
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
def case(name: str, body: str, *, slice_name: str, family: str) -> PairCase:
|
||||
@@ -210,7 +210,7 @@ def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_pa
|
||||
|
||||
def test_the_ruleset_generator_is_measured_too_so_the_comparison_is_reproducible(tmp_path: Path) -> None:
|
||||
"""12.1% against 96.6% is why the ruleset is not the generator. A remembered number is not a measurement."""
|
||||
from openultrasast.learning.candidates import ceiling
|
||||
from openultrasast.model.candidates import ceiling
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_slice, select_vendored
|
||||
|
||||
cases = [c for c in select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py")) if not c.unscorable][:6]
|
||||
@@ -225,12 +225,12 @@ def test_the_command_runs_offline_and_writes_a_stable_artifact(tmp_path: Path, m
|
||||
|
||||
for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
out = tmp_path / "learning"
|
||||
assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0
|
||||
out = tmp_path / "model"
|
||||
assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0
|
||||
artifact = out / "candidate-ceiling.json"
|
||||
assert artifact.is_file()
|
||||
first = artifact.read_bytes()
|
||||
assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0
|
||||
assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0
|
||||
assert artifact.read_bytes() == first, "the ceiling is a property of the corpus, not of when it was measured"
|
||||
import json
|
||||
|
||||
|
||||
@@ -44,71 +44,6 @@ def test_the_baseline_records_where_it_came_from() -> None:
|
||||
# --- one optional dependency at a time ---------------------------------------
|
||||
|
||||
|
||||
def test_without_harnessx_the_meta_agent_proposer_proposes_nothing_and_says_why(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning import proposer as proposer_module
|
||||
from openultrasast.learning.detectors import FamilyConfig
|
||||
from openultrasast.learning.proposer import FailureFacts, HarnessXProposer
|
||||
|
||||
monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False)
|
||||
(tmp_path / "injection").mkdir()
|
||||
facts = FailureFacts(
|
||||
family="injection",
|
||||
config=FamilyConfig(
|
||||
family="injection", version="0", prompt="p", checklist="c", tools=("read_file",), max_steps=1, max_cost_usd=0.1, max_chars=100
|
||||
),
|
||||
)
|
||||
proposer = HarnessXProposer(configs_dir=tmp_path, model="m")
|
||||
assert proposer.propose(facts) is None
|
||||
assert proposer.reason == "harnessx_unavailable"
|
||||
|
||||
|
||||
def test_without_a_chat_endpoint_the_learning_run_records_it_and_still_writes_its_artifact(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from openultrasast.cli import main
|
||||
|
||||
for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
(tmp_path / "a-v.py").write_text("def run():\n return 1\n")
|
||||
(tmp_path / "a-f.py").write_text("def run():\n return 1\n")
|
||||
(tmp_path / "pairs.toml").write_text(
|
||||
'[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n'
|
||||
'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n'
|
||||
)
|
||||
printed: list[str] = []
|
||||
monkeypatch.setattr("builtins.print", lambda *args, **kwargs: printed.append(" ".join(str(item) for item in args)))
|
||||
assert main(["learning", "baseline", "--catalog", str(tmp_path / "pairs.toml"), "--out", str(tmp_path / "l"), "--model", "none"]) == 0
|
||||
assert any("learning_endpoint_unavailable" in line for line in printed)
|
||||
assert (tmp_path / "l" / "baseline" / "none" / "report.json").is_file()
|
||||
|
||||
|
||||
def test_without_family_configurations_a_scan_records_the_reason_rather_than_the_traceback(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from openultrasast.cli import _run_family_detectors
|
||||
|
||||
class _Settings:
|
||||
configs_dir = "does/not/exist"
|
||||
families_path = None
|
||||
|
||||
class _Runtime:
|
||||
def __init__(self) -> None:
|
||||
self.state: dict[str, list[dict[str, object]]] = {"degradations": []}
|
||||
|
||||
runtime = _Runtime()
|
||||
findings, payload, cited = _run_family_detectors(
|
||||
root=tmp_path,
|
||||
hotspots=[],
|
||||
cases=None,
|
||||
settings=_Settings(),
|
||||
config=None,
|
||||
runtime=runtime, # type: ignore[arg-type]
|
||||
)
|
||||
assert (findings, payload, cited) == ([], {}, {})
|
||||
assert [item["reason"] for item in runtime.state["degradations"]] == ["learning_configs_unavailable"]
|
||||
|
||||
|
||||
def test_without_the_network_a_pointer_pair_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.pairs import evaluate_catalog, load_pair_catalog, select_slice
|
||||
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
"""A unit fixture may not be easier than the corpus it claims to cover (learning-harness 4.2, Req 10.3, 10.4).
|
||||
|
||||
Two things a corpus number cannot survive: a byte-identical pair counted as a detector result, and a fixture that
|
||||
passes because the world it describes is simpler than the one the detector meets. The first is excluded by name;
|
||||
the second is measured on three axes the corpus actually varies on, and the comparison is recorded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_obligation_operations import APP # noqa: E402
|
||||
|
||||
from openultrasast.learning.difficulty import DifficultyVector, difficulty_of, hardest, strictly_easier # noqa: E402
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored # noqa: E402
|
||||
|
||||
TWINS = {
|
||||
"dsvw-dsvw-do-get",
|
||||
"python-insecure-app-main-try-hack-me",
|
||||
"pythonssti-main-read-root",
|
||||
"vampi-books-get-by-title",
|
||||
"vulnpy-deserialization-do-pickle-load",
|
||||
}
|
||||
# The rows the shared obligations fixture claims to cover: the vibe-py absence pairs.
|
||||
COVERED = ("vampi-books-get-by-title", "vampi-users-update-password", "threatbyte-api-v1-get", "threatbyte-api-v1-delete")
|
||||
FIXTURE_HANDLERS = ("guarded", "constrained", "constrained_too", "leaky", "from_spec")
|
||||
REPORT = Path("benchmarks/measurements/2026-09-06-fixture-difficulty.json")
|
||||
|
||||
|
||||
# --- identical twins ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_identical_twin_declares_the_limit_rather_than_only_hashing_to_it() -> None:
|
||||
"""A content hash is evidence; the catalog row is the maintainer standing behind it.
|
||||
|
||||
A re-harvest that changes one byte would otherwise turn a known-unscorable row back into a scored one with no
|
||||
review, and the number would move for a reason nobody chose."""
|
||||
cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))}
|
||||
for name in sorted(TWINS):
|
||||
assert cases[name].known_limit == "identical_twin", f"{name} is a twin by content but says nothing in the catalog"
|
||||
assert cases[name].unscorable == "identical_twin", "the declared limit and the computed one must be one reason, not two"
|
||||
|
||||
|
||||
def test_the_twins_leave_every_denominator_with_their_reason_listed() -> None:
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.scoring import PairFamilyScore, aggregate
|
||||
|
||||
cases = [case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if case.name in TWINS]
|
||||
scores = [
|
||||
PairFamilyScore(
|
||||
pair=case.name, family="injection", slice=case.slice, runs=(), outcome="unscorable", unscorable_reason=case.unscorable
|
||||
)
|
||||
for case in cases
|
||||
]
|
||||
metrics = aggregate(scores, taxonomy=load_families())["injection"]
|
||||
assert metrics.scorable == 0
|
||||
assert metrics.unscorable == {"identical_twin": len(TWINS)}
|
||||
|
||||
|
||||
# --- the difficulty check ----------------------------------------------------
|
||||
|
||||
|
||||
def _corpus_vectors() -> list[DifficultyVector]:
|
||||
cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))}
|
||||
vectors = []
|
||||
for name in COVERED:
|
||||
case = cases[name]
|
||||
function = next(row.function for row in case.expected if row.function)
|
||||
context = [vuln.read_text() for _relpath, vuln, _fixed in case.context_files]
|
||||
vectors.append(difficulty_of(case.vuln_file.read_text(), function, context=context))
|
||||
return vectors
|
||||
|
||||
|
||||
def test_the_shared_obligations_fixture_is_not_easier_than_the_rows_it_covers() -> None:
|
||||
"""The check Req 10.4 asks for, on the fixture every obligations test shares."""
|
||||
fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS]
|
||||
corpus = _corpus_vectors()
|
||||
assert not strictly_easier(fixture, corpus), (
|
||||
f"the fixture reaches {hardest(fixture)} where the corpus reaches {hardest(corpus)} "
|
||||
"(binding hops, registration distance, guard distance)"
|
||||
)
|
||||
|
||||
|
||||
def test_the_check_catches_a_fixture_that_is_easier_on_one_axis(tmp_path: Path) -> None:
|
||||
"""Mutation of the check itself: drop the corpus-shaped handler and it has to fail."""
|
||||
without = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS if name != "from_spec"]
|
||||
assert strictly_easier(without, _corpus_vectors())
|
||||
assert hardest(without)[1] == 0, "every remaining handler carries its route decorator; nothing is registered elsewhere"
|
||||
|
||||
|
||||
def test_the_verdict_is_recorded_beside_the_numbers_it_explains() -> None:
|
||||
assert REPORT.is_file(), "the fixture-difficulty verdict is committed, not recomputed from memory"
|
||||
payload = json.loads(REPORT.read_text())
|
||||
fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS]
|
||||
corpus = _corpus_vectors()
|
||||
assert payload["fixture"] == [row.to_dict() for row in fixture]
|
||||
assert payload["corpus"] == [row.to_dict() for row in corpus]
|
||||
assert payload["strictly_easier"] is False and payload["verdict"] == "fixture_matches_corpus"
|
||||
assert payload["axes"] == ["binding_hops", "registration_distance", "guard_distance"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
[
|
||||
("@app.route('/x')\ndef h():\n return Book.query.filter_by(user_id=request.user.id).first()\n", (0, 0, 2)),
|
||||
("@app.route('/x')\n@login_required\ndef h():\n return 1\n", (0, 0, 0)),
|
||||
("def h():\n u = current_user.id\n v = u\n return v\n", (2, 2, 1)),
|
||||
("app.add_url_rule('/x', view_func=h)\n\n\ndef h():\n return 1\n", (0, 1, 2)),
|
||||
],
|
||||
)
|
||||
def test_each_axis_is_read_off_the_code(source: str, expected: tuple[int, int, int]) -> None:
|
||||
assert difficulty_of(source, "h").axes() == expected
|
||||
@@ -15,7 +15,6 @@ from openultrasast.index import (
|
||||
write_vector_index,
|
||||
)
|
||||
from openultrasast.preprocess import RepoSnapshot, preprocess_repository
|
||||
from openultrasast.vectorstore import bakeoff_json_local, bakeoff_payload
|
||||
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
@@ -103,15 +102,3 @@ def test_retrieval_package_is_bounded_by_character_budget() -> None:
|
||||
assert package.truncated is True
|
||||
|
||||
|
||||
def test_json_local_bakeoff_selects_default_store(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def login(user):\n return user\n")
|
||||
snapshot, targets = preprocess_repository(repo)
|
||||
chunks = build_code_chunks(repo, targets)
|
||||
index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient())
|
||||
|
||||
result = bakeoff_json_local(index, tmp_path / "vector-index.json", query_embedding=[20.0, 1.0])
|
||||
|
||||
assert result.passed is True
|
||||
assert bakeoff_payload(result)["selected_default"] is True
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
"""learning-harness task 3.6: the acceptance rule and the directory revert (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learning.rounds import NoiseFloor
|
||||
|
||||
|
||||
def _floor(family: str, budget: int = 0, *, gates: bool = True) -> NoiseFloor:
|
||||
return NoiseFloor(family=family, k_runs=5, pairs=10, flaky_pairs=budget, negative_flip_rate=0.0, budget_flips=budget, gates=gates)
|
||||
|
||||
|
||||
def _decide(train: float, holdout: float, sweep: dict[str, int], floors: dict[str, NoiseFloor]): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.acceptance import decide
|
||||
|
||||
return decide(target="access_control", train_delta=train, holdout_delta=holdout, sweep=sweep, floors=floors)
|
||||
|
||||
|
||||
def test_a_change_that_helps_and_hurts_nobody_is_accepted() -> None:
|
||||
verdict = _decide(0.1, 0.2, {"injection": 0}, {"injection": _floor("injection")})
|
||||
assert verdict.accepted and verdict.reason == "improved"
|
||||
|
||||
|
||||
def test_one_side_may_stand_still_but_the_other_must_move() -> None:
|
||||
assert _decide(0.0, 0.2, {}, {}).accepted
|
||||
assert _decide(0.2, 0.0, {}, {}).accepted
|
||||
tie = _decide(0.0, 0.0, {}, {})
|
||||
assert not tie.accepted and tie.reason == "tie" # a tie is rejected, or the loop drifts on noise
|
||||
|
||||
|
||||
def test_a_decrease_on_either_side_is_rejected() -> None:
|
||||
train_down = _decide(-0.1, 0.5, {}, {})
|
||||
assert not train_down.accepted and train_down.reason == "train_regressed"
|
||||
holdout_down = _decide(0.5, -0.1, {}, {})
|
||||
assert not holdout_down.accepted and holdout_down.reason == "holdout_regressed"
|
||||
|
||||
|
||||
def test_another_family_beyond_its_measured_budget_rejects_the_round() -> None:
|
||||
within = _decide(0.2, 0.2, {"injection": 2}, {"injection": _floor("injection", 2)})
|
||||
assert within.accepted # inside the noise this family shows against itself
|
||||
beyond = _decide(0.2, 0.2, {"injection": 3}, {"injection": _floor("injection", 2)})
|
||||
assert not beyond.accepted and beyond.reason == "collateral_regression"
|
||||
assert beyond.offenders == ("injection",)
|
||||
|
||||
|
||||
def test_a_family_that_never_gates_cannot_reject_a_round() -> None:
|
||||
verdict = _decide(0.2, 0.2, {"memory": 99}, {"memory": _floor("memory", 0, gates=False)})
|
||||
assert verdict.accepted and verdict.offenders == ()
|
||||
|
||||
|
||||
def test_a_family_with_no_measured_floor_gets_no_budget() -> None:
|
||||
verdict = _decide(0.2, 0.2, {"path": 1}, {})
|
||||
assert not verdict.accepted and verdict.offenders == ("path",) # unmeasured is not the same as unlimited
|
||||
|
||||
|
||||
def test_a_snapshot_restores_a_family_directory_byte_for_byte(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.acceptance import DirectorySnapshot
|
||||
|
||||
directory = tmp_path / "access_control"
|
||||
directory.mkdir()
|
||||
(directory / "prompt.md").write_text("original\n")
|
||||
(directory / "family.toml").write_text('version = "0"\n')
|
||||
snapshot = DirectorySnapshot.of(directory)
|
||||
(directory / "prompt.md").write_text("changed\n")
|
||||
(directory / "extra.md").write_text("added by a round\n")
|
||||
(directory / "family.toml").unlink()
|
||||
snapshot.restore()
|
||||
assert (directory / "prompt.md").read_text() == "original\n"
|
||||
assert (directory / "family.toml").read_text() == 'version = "0"\n'
|
||||
assert not (directory / "extra.md").exists() # a file the round added is not left behind
|
||||
assert sorted(path.name for path in directory.iterdir()) == ["family.toml", "prompt.md"]
|
||||
|
||||
|
||||
def test_every_other_family_directory_is_untouched_across_a_round(tmp_path: Path) -> None:
|
||||
"""Req 6.2: a round can only write its target, so the freeze is structural rather than promised."""
|
||||
from openultrasast.learning.acceptance import DirectorySnapshot, digest_configs
|
||||
from openultrasast.learning.detectors import write_default_configs
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.proposer import Proposal, apply_proposal
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
before = digest_configs(configs)
|
||||
snapshot = DirectorySnapshot.of(configs / "access_control")
|
||||
apply_proposal(Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}), configs / "access_control")
|
||||
after = digest_configs(configs)
|
||||
assert after["access_control"] != before["access_control"]
|
||||
assert {name: value for name, value in after.items() if name != "access_control"} == {
|
||||
name: value for name, value in before.items() if name != "access_control"
|
||||
}
|
||||
snapshot.restore()
|
||||
assert digest_configs(configs) == before # a rejected round leaves the whole tree as it found it
|
||||
|
||||
|
||||
def test_a_bumped_version_is_written_only_on_acceptance(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.acceptance import bump_version
|
||||
from openultrasast.learning.detectors import load_family_configs, write_default_configs
|
||||
from openultrasast.learning.families import load_families
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
bump_version(configs / "access_control", "1")
|
||||
taxonomy = load_families()
|
||||
assert load_family_configs(configs, taxonomy)["access_control"].version == "1"
|
||||
assert load_family_configs(configs, taxonomy)["injection"].version == "0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("spent", "cap", "expected"), [(0.5, 10.0, True), (10.0, 10.0, True), (10.01, 10.0, False)])
|
||||
def test_a_round_over_its_cost_cap_is_reverted(spent: float, cap: float, expected: bool) -> None:
|
||||
from openultrasast.learning.acceptance import within_budget
|
||||
|
||||
assert within_budget(spent, cap) is expected
|
||||
@@ -1,116 +0,0 @@
|
||||
"""learning-harness task 2.4: aggregation, noise handling and honest denominators (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.scoring import FamilyOutcome, PairFamilyScore
|
||||
|
||||
|
||||
def _score(
|
||||
pair: str, outcome: FamilyOutcome, *, family: str = "access_control", slice_name: str = "vibe-py", **extra: object
|
||||
) -> PairFamilyScore:
|
||||
runs = () if outcome == "unscorable" else (outcome,)
|
||||
return PairFamilyScore(pair=pair, family=family, slice=slice_name, runs=runs, outcome=outcome, **extra) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_rates_use_only_the_scorable_rows_and_the_reasons_are_kept() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
taxonomy = load_families()
|
||||
scores = [
|
||||
_score("a", "pair_correct"),
|
||||
_score("b", "both_silent"),
|
||||
_score("c", "unscorable", unscorable_reason="identical_twin"),
|
||||
_score("d", "unscorable", unscorable_reason="identical_twin"),
|
||||
_score("e", "unscorable", unscorable_reason="unsupported_language"),
|
||||
]
|
||||
metrics = aggregate(scores, taxonomy=taxonomy)["access_control"]
|
||||
assert metrics.scorable == 2 and metrics.unscorable == {"identical_twin": 2, "unsupported_language": 1}
|
||||
assert metrics.recall == 0.5 and metrics.silence == 1.0 and metrics.youden == 0.5
|
||||
assert metrics.outcomes == {"pair_correct": 1, "both_silent": 1}
|
||||
assert metrics.taxonomy_version == taxonomy.version
|
||||
json.dumps(metrics.to_dict())
|
||||
|
||||
|
||||
def test_a_family_with_nothing_scorable_reports_no_rate_rather_than_a_perfect_one() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
metrics = aggregate([_score("a", "unscorable", unscorable_reason="identical_twin")], taxonomy=load_families())["access_control"]
|
||||
assert metrics.scorable == 0 and metrics.recall == 0.0 and metrics.silence == 0.0 and metrics.youden == 0.0
|
||||
|
||||
|
||||
def test_directional_bias_separates_flagging_everything_from_saying_everything_is_safe() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
taxonomy = load_families()
|
||||
loud = aggregate([_score("a", "both_flagged"), _score("b", "both_flagged")], taxonomy=taxonomy)["access_control"]
|
||||
quiet = aggregate([_score("a", "both_silent"), _score("b", "both_silent")], taxonomy=taxonomy)["access_control"]
|
||||
assert loud.directional_bias == 1.0 and quiet.directional_bias == -1.0
|
||||
assert loud.youden == quiet.youden == 0.0 # the same Youden, opposite failures: the index is what tells them apart
|
||||
|
||||
|
||||
def test_recall_at_a_fixed_false_positive_ceiling_is_forfeited_when_the_family_leaks() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
taxonomy = load_families()
|
||||
clean = [_score(str(index), "pair_correct") for index in range(9)] + [_score("x", "both_silent")]
|
||||
assert aggregate(clean, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"].fixed_fpr_recall == 0.9
|
||||
leaky = [_score(str(index), "pair_correct") for index in range(8)] + [_score("y", "both_flagged"), _score("z", "reversed")]
|
||||
metrics = aggregate(leaky, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"]
|
||||
assert metrics.recall == 0.9 and metrics.fixed_fpr_recall == 0.0 # a 20% leak rate buys no credit at a 10% ceiling
|
||||
|
||||
|
||||
def test_negative_flips_and_the_reliable_change_test_compare_against_a_baseline() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
taxonomy = load_families()
|
||||
baseline = [_score(str(index), "pair_correct") for index in range(10)]
|
||||
same = aggregate(baseline, taxonomy=taxonomy, baseline=baseline)["access_control"]
|
||||
assert same.negative_flips == 0 and same.reliable_change is False
|
||||
broke_one = [_score("0", "both_silent")] + [_score(str(index), "pair_correct") for index in range(1, 10)]
|
||||
one = aggregate(broke_one, taxonomy=taxonomy, baseline=baseline)["access_control"]
|
||||
assert one.negative_flips == 1 and one.reliable_change is False # one flip in ten is inside the noise
|
||||
broke_six = [_score(str(index), "both_silent") for index in range(6)] + [_score(str(index), "pair_correct") for index in range(6, 10)]
|
||||
six = aggregate(broke_six, taxonomy=taxonomy, baseline=baseline)["access_control"]
|
||||
assert six.negative_flips == 6 and six.reliable_change is True
|
||||
assert six.p_value is not None and six.p_value < 0.05
|
||||
|
||||
|
||||
def test_metrics_are_reported_per_slice_when_asked() -> None:
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
taxonomy = load_families()
|
||||
scores = [_score("a", "pair_correct", slice_name="vibe-py"), _score("b", "both_silent", slice_name="agent-vfc")]
|
||||
overall = aggregate(scores, taxonomy=taxonomy)
|
||||
assert set(overall) == {"access_control"} and overall["access_control"].scorable == 2
|
||||
per_slice = aggregate(scores, taxonomy=taxonomy, per_slice=True)
|
||||
assert set(per_slice) == {"vibe-py/access_control", "agent-vfc/access_control"}
|
||||
assert per_slice["vibe-py/access_control"].recall == 1.0 and per_slice["agent-vfc/access_control"].recall == 0.0
|
||||
|
||||
|
||||
def test_the_sign_test_is_two_sided_and_symmetric() -> None:
|
||||
from openultrasast.learning.scoring import sign_test
|
||||
|
||||
assert sign_test(0, 0) == 1.0
|
||||
assert sign_test(6, 0) == sign_test(0, 6) < 0.05
|
||||
assert sign_test(1, 0) > 0.05 and sign_test(3, 3) == 1.0
|
||||
|
||||
|
||||
def test_every_family_figure_says_when_hierarchical_credit_could_not_apply() -> None:
|
||||
"""Req 3.7: with a flat taxonomy no answer can earn partial credit, and the number must carry that."""
|
||||
from openultrasast.learning.families import Family, FamilyTaxonomy
|
||||
from openultrasast.learning.scoring import aggregate
|
||||
|
||||
flat = load_families()
|
||||
metrics = aggregate([_score("a", "pair_correct")], taxonomy=flat)["access_control"]
|
||||
assert metrics.hierarchical_credit is False and metrics.to_dict()["hierarchical_credit"] is False
|
||||
nested = FamilyTaxonomy(
|
||||
version="test",
|
||||
families=(
|
||||
Family(id="injection", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none"),
|
||||
Family(id="access_control", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none", parent="injection"),
|
||||
),
|
||||
)
|
||||
assert aggregate([_score("a", "pair_correct")], taxonomy=nested)["access_control"].hierarchical_credit is True
|
||||
@@ -1,197 +0,0 @@
|
||||
"""learning-harness task 2.7: canary templates and the second judge (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.config import SandboxConfig
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.regress.safety import check_snippet_safety
|
||||
from openultrasast.sandbox import SandboxResult
|
||||
from openultrasast.sandbox.runner import FakeSandboxRunner
|
||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient
|
||||
|
||||
CANARY_FAMILIES = ("injection", "path", "deserialization")
|
||||
|
||||
|
||||
def _claim(family: str, function: str = "lookup", line: int = 3) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"detector:app.py:{line}",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name=function,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def test_every_canary_template_renders_and_passes_the_snippet_safety_check() -> None:
|
||||
from openultrasast.learning.canaries import render_canary
|
||||
|
||||
for family in CANARY_FAMILIES:
|
||||
snippet, token = render_canary(family, language="python", path="app.py", function="lookup")
|
||||
assert token and token in snippet
|
||||
check_snippet_safety(snippet) # raises when the template would be refused before it ever runs
|
||||
assert "/workspace" in snippet # the candidate is imported from the read-only source mount
|
||||
assert render_canary("memory", language="python", path="app.py", function="f") is None
|
||||
assert render_canary("injection", language="rust", path="a.rs", function="f") is None
|
||||
|
||||
|
||||
def test_a_canary_token_is_unique_to_the_experiment_it_names() -> None:
|
||||
from openultrasast.learning.canaries import render_canary
|
||||
|
||||
first = render_canary("injection", language="python", path="app.py", function="lookup")
|
||||
second = render_canary("injection", language="python", path="app.py", function="other")
|
||||
third = render_canary("path", language="python", path="app.py", function="lookup")
|
||||
assert first and second and third
|
||||
assert first[1] != second[1] != third[1] and first[1] != third[1]
|
||||
# the same family, file and function is the same experiment, so it is deliberately the same token
|
||||
assert render_canary("injection", language="python", path="app.py", function="lookup")[1] == first[1] # type: ignore[index]
|
||||
|
||||
|
||||
def test_the_canary_verifier_proves_only_when_the_oracle_fires(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.canaries import CanaryVerifier
|
||||
|
||||
(tmp_path / "app.py").write_text("def lookup(value):\n return value\n")
|
||||
fired = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, f"CANARY {token}", "", False)), limits=SandboxConfig())
|
||||
proven = fired.verify(_claim("injection"), root=tmp_path, language="python")
|
||||
assert proven.rung == "proven" and proven.reason == "oracle_fired"
|
||||
quiet = CanaryVerifier(runner=_runner(lambda _t: SandboxResult(1, "boom", "Traceback", False)), limits=SandboxConfig())
|
||||
unproven = quiet.verify(_claim("injection"), root=tmp_path, language="python")
|
||||
assert unproven.rung == "suspicion" and unproven.reason == "oracle_quiet" # a crash is not this family's evidence
|
||||
|
||||
|
||||
def test_a_family_or_language_with_no_template_stays_at_suspicion(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.canaries import CanaryVerifier
|
||||
|
||||
(tmp_path / "app.py").write_text("def lookup(value):\n return value\n")
|
||||
verifier = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig())
|
||||
assert verifier.verify(_claim("memory"), root=tmp_path, language="python").reason == "no_template"
|
||||
assert verifier.verify(_claim("injection"), root=tmp_path, language="rust").reason == "no_template"
|
||||
|
||||
|
||||
def test_the_registry_uses_the_canary_verifier_when_one_is_supplied(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.canaries import CanaryVerifier
|
||||
from openultrasast.learning.verifiers import NoVerifier, verifier_for
|
||||
|
||||
taxonomy = load_families()
|
||||
canary = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig())
|
||||
assert verifier_for(taxonomy.by_id("injection"), canary=canary) is canary
|
||||
assert isinstance(verifier_for(taxonomy.by_id("access_control"), canary=canary), type(verifier_for(taxonomy.by_id("access_control"))))
|
||||
assert isinstance(verifier_for(taxonomy.by_id("memory"), canary=canary), NoVerifier)
|
||||
|
||||
|
||||
def test_nothing_is_published_as_proven_until_a_second_judge_confirms() -> None:
|
||||
from openultrasast.learning.canaries import confirm_proven
|
||||
from openultrasast.learning.verifiers import Verification
|
||||
|
||||
proven = Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc")
|
||||
agrees = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "the row came back"}))])
|
||||
kept = confirm_proven(proven, _claim("injection"), client=agrees, model="judge", snippet="print('x')")
|
||||
assert kept.rung == "proven"
|
||||
declines = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": False, "reason": "unrelated output"}))])
|
||||
dropped = confirm_proven(proven, _claim("injection"), client=declines, model="judge", snippet="print('x')")
|
||||
assert dropped.rung == "static_corroboration" and dropped.reason == "judge_declined"
|
||||
assert confirm_proven(proven, _claim("injection"), client=None, model="", snippet="x").rung == "static_corroboration"
|
||||
lower = Verification(rung="suspicion", reason="oracle_quiet")
|
||||
assert confirm_proven(lower, _claim("injection"), client=agrees, model="judge", snippet="x") is lower # nothing to confirm
|
||||
|
||||
|
||||
def test_the_judge_sees_the_snippet_and_the_oracle_output_and_never_the_label() -> None:
|
||||
from openultrasast.learning.canaries import confirm_proven
|
||||
from openultrasast.learning.verifiers import Verification
|
||||
|
||||
client = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "ok"}))])
|
||||
confirm_proven(
|
||||
Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc"),
|
||||
_claim("injection"),
|
||||
client=client,
|
||||
model="judge",
|
||||
snippet="print('CANARY abc')",
|
||||
)
|
||||
prompt = json.dumps(client.calls[0]["messages"])
|
||||
assert "CANARY abc" in prompt and "print('CANARY abc')" in prompt
|
||||
assert client.calls[0].get("json_object") is True and client.calls[0]["tools"] == []
|
||||
assert "family:injection" not in prompt # the judge is told the claim, not the answer key
|
||||
|
||||
|
||||
def test_templates_and_oracle_code_live_outside_every_proposer_write_root(tmp_path: Path) -> None:
|
||||
from openultrasast.learning import canaries
|
||||
from openultrasast.learning.detectors import write_default_configs
|
||||
|
||||
write_roots = set(write_default_configs(tmp_path, load_families(), prompt="p", version="0").values())
|
||||
module = Path(canaries.__file__).resolve()
|
||||
assert not any(module.is_relative_to(root.resolve()) for root in write_roots)
|
||||
assert "site-packages" in str(module) or "src/openultrasast" in str(module)
|
||||
|
||||
|
||||
def _runner(reply: object) -> FakeSandboxRunner:
|
||||
"""A sandbox that answers from whatever canary token the snippet actually planted."""
|
||||
|
||||
class _Runner(FakeSandboxRunner): # type: ignore[misc]
|
||||
def run(self, spec: object) -> SandboxResult:
|
||||
body = "\n".join(str(value) for value in (getattr(spec, "scratch_files", {}) or {}).values())
|
||||
marker = next((word.strip("'\"") for word in body.split() if "ousast-canary-" in word), "ousast-canary-none")
|
||||
return reply(marker) # type: ignore[operator]
|
||||
|
||||
return _Runner()
|
||||
|
||||
|
||||
VULNERABLE = {
|
||||
"injection": "import os\n\n\ndef lookup(value):\n os.system(value)\n return 'done'\n",
|
||||
"path": "def lookup(value):\n with open(value, encoding='utf-8') as handle:\n return handle.read()\n",
|
||||
"deserialization": "import pickle\n\n\ndef lookup(value):\n return pickle.loads(value)\n",
|
||||
}
|
||||
FIXED = {
|
||||
"injection": (
|
||||
"import shlex, subprocess\n\n\ndef lookup(value):\n"
|
||||
" subprocess.run(['echo', shlex.quote(value)], check=False)\n return 'done'\n"
|
||||
),
|
||||
"path": (
|
||||
"from pathlib import Path\n\n\ndef lookup(value):\n"
|
||||
" name = Path(value).name\n return (Path('/tmp') / name).read_text(encoding='utf-8') if False else ''\n"
|
||||
),
|
||||
"deserialization": "import json\n\n\ndef lookup(value):\n return json.loads(value.decode('utf-8', 'ignore') or '{}')\n",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("family", CANARY_FAMILIES)
|
||||
def test_each_canary_fires_on_a_real_bug_and_stays_quiet_on_its_fixed_twin(tmp_path: Path, family: str) -> None:
|
||||
"""The oracle must be a property of the code, not of the harness: run the snippet for real, both sides.
|
||||
|
||||
The injection fixture's fixed twin deliberately echoes its input, which is why the payload carries a
|
||||
value unrelated to the oracle token: an echoing handler must not be able to satisfy the oracle.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from openultrasast.learning.canaries import render_canary
|
||||
|
||||
for side, sources in (("vuln", VULNERABLE), ("fixed", FIXED)):
|
||||
workspace = tmp_path / side / "workspace"
|
||||
scratch = tmp_path / side / "scratch"
|
||||
workspace.mkdir(parents=True)
|
||||
scratch.mkdir(parents=True)
|
||||
(workspace / "app.py").write_text(sources[family])
|
||||
rendered = render_canary(
|
||||
family, language="python", path="app.py", function="lookup", workspace=str(workspace), scratch=str(scratch)
|
||||
)
|
||||
assert rendered is not None
|
||||
snippet, token = rendered
|
||||
script = scratch / "case.py"
|
||||
script.write_text(snippet)
|
||||
completed = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, timeout=60, check=False)
|
||||
fired = token in completed.stdout or token in completed.stderr
|
||||
assert fired is (side == "vuln"), f"{family} {side}: {completed.stdout!r} {completed.stderr!r}"
|
||||
@@ -1,115 +0,0 @@
|
||||
"""learning-harness task 2.2: the classifier reports on itself and queues disagreements (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, cwe: str, mechanism: str = "other", family: str | None = None, tier: str = "reviewed") -> PairCase:
|
||||
(tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}.py",
|
||||
fixed_file=tmp_path / f"{name}.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function="f", mechanism=mechanism, family=family),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
review_tier=tier,
|
||||
reviewer="m" if tier == "reviewed" else "",
|
||||
)
|
||||
|
||||
|
||||
def test_the_report_covers_the_whole_corpus_and_names_the_unknowns(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import measure_classifier
|
||||
|
||||
taxonomy = load_families()
|
||||
cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG))
|
||||
report = measure_classifier(cases, taxonomy, client=None)
|
||||
assert report.taxonomy_version == taxonomy.version
|
||||
assert report.classified == len(cases)
|
||||
assert sum(report.per_tier.values()) == len(cases) and set(report.per_tier) <= {"static", "model", "none"}
|
||||
assert report.per_family["memory"] >= 150 # the vfc rows land in one family instead of reading `other`
|
||||
assert report.per_family["access_control"] >= 10 and report.per_family["injection"] >= 40
|
||||
assert report.unknown == 2 and report.per_tier["static"] == len(cases) - 2 # tier one places all but two rows
|
||||
payload = report.to_dict()
|
||||
json.dumps(payload)
|
||||
# Task 4.1 declared `family` on the ten absence rows; only the four at a reference tier (`seeded`) count as labels
|
||||
# a maintainer stands behind, and the six `title`-tier agent rows do not. Agreement over four rows of one family is
|
||||
# not yet a measurement of the classifier, so the number is recorded, not celebrated.
|
||||
assert payload["labeled"] == 4 and payload["agreement"] == 1.0
|
||||
assert set(report.confusion) == {"access_control"}
|
||||
|
||||
|
||||
def test_agreement_uses_the_first_answer_and_the_oracle_uses_any_of_them(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import measure_classifier
|
||||
|
||||
taxonomy = load_families()
|
||||
cases = [
|
||||
_case(tmp_path, "hit", cwe="CWE-89", family="injection"), # tier one answers injection: agreement and oracle
|
||||
_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), # answers injection: neither
|
||||
]
|
||||
report = measure_classifier(cases, taxonomy, client=None)
|
||||
assert report.labeled == 2
|
||||
assert report.agreement == 0.5 and report.oracle_router == 0.5
|
||||
assert abs(report.random_router - 1 / 9) < 1e-9 # a uniform pick over the nine real families
|
||||
assert report.confusion["injection"]["injection"] == 1 and report.confusion["access_control"]["injection"] == 1
|
||||
|
||||
|
||||
def test_every_disagreement_is_queued_once_with_both_answers_and_the_tier(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import measure_classifier, write_review_queue
|
||||
|
||||
taxonomy = load_families()
|
||||
cases = [_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), _case(tmp_path, "hit", cwe="CWE-89", family="injection")]
|
||||
report = measure_classifier(cases, taxonomy, client=None)
|
||||
assert [entry.pair for entry in report.queue] == ["miss"]
|
||||
entry = report.queue[0]
|
||||
assert entry.label == "access_control" and entry.classified == ("injection",) and entry.tier == "static"
|
||||
queue_path = tmp_path / "review-queue.jsonl"
|
||||
write_review_queue(queue_path, report.queue)
|
||||
write_review_queue(queue_path, report.queue) # a second measurement must not duplicate the row
|
||||
rows = [json.loads(line) for line in queue_path.read_text().splitlines() if line.strip()]
|
||||
assert [row["pair"] for row in rows] == ["miss"]
|
||||
assert rows[0]["label"] == "access_control" and rows[0]["classified"] == ["injection"] and rows[0]["tier"] == "static"
|
||||
|
||||
|
||||
def test_measuring_never_writes_a_label(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import measure_classifier
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path, "miss", cwe="CWE-89", family="access_control")
|
||||
measure_classifier([case], taxonomy, client=None)
|
||||
assert case.expected[0].family == "access_control" # the classifier proposes; the maintainer decides
|
||||
|
||||
|
||||
def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import measure_classifier
|
||||
|
||||
taxonomy = load_families()
|
||||
cases = [
|
||||
_case(tmp_path, "trusted", cwe="CWE-89", family="access_control", tier="reviewed"),
|
||||
_case(tmp_path, "untrusted", cwe="CWE-89", family="access_control", tier="title"),
|
||||
]
|
||||
report = measure_classifier(cases, taxonomy, client=None)
|
||||
assert report.labeled == 1 and report.classified == 2 # an unreviewed label is not a reference answer
|
||||
assert [entry.pair for entry in report.queue] == ["trusted"]
|
||||
|
||||
|
||||
def test_the_report_says_when_hierarchical_credit_could_not_apply(tmp_path: Path) -> None:
|
||||
"""Req 2.6: the shipped taxonomy is flat, so partial credit is inert and every figure must say so."""
|
||||
from openultrasast.learning.classify import measure_classifier
|
||||
|
||||
taxonomy = load_families()
|
||||
report = measure_classifier([_case(tmp_path, "a", cwe="CWE-89", family="injection")], taxonomy, client=None)
|
||||
assert report.hierarchical_credit is False
|
||||
assert report.to_dict()["hierarchical_credit"] is False
|
||||
@@ -1,163 +0,0 @@
|
||||
"""learning-harness task 2.1: the three-tier auto-classifier (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import PairCase
|
||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient
|
||||
|
||||
APP = (
|
||||
"from flask import request\n\n\n"
|
||||
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
|
||||
" return Book.query.filter_by(book_title=title).first()\n\n\n"
|
||||
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
|
||||
" owner = request.args.get('owner')\n"
|
||||
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n"
|
||||
)
|
||||
|
||||
|
||||
def _case(tmp_path: Path, **row: object) -> PairCase:
|
||||
(tmp_path / "v.py").write_text(APP)
|
||||
(tmp_path / "f.py").write_text(APP + "# fixed\n")
|
||||
expected = ExpectedFinding(
|
||||
cwe=str(row.get("cwe", "")),
|
||||
vulnerability_class="x",
|
||||
path="app.py",
|
||||
evidence="",
|
||||
function="leaky",
|
||||
mechanism=str(row.get("mechanism", "other")),
|
||||
family=row.get("family"), # type: ignore[arg-type]
|
||||
)
|
||||
return PairCase(
|
||||
name="t",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "f.py",
|
||||
relpath="app.py",
|
||||
expected=(expected,),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
)
|
||||
|
||||
|
||||
def test_tier_one_decides_from_a_declared_family_a_mechanism_or_a_weakness(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_pair
|
||||
|
||||
taxonomy = load_families()
|
||||
declared = classify_pair(_case(tmp_path, family="injection", cwe="CWE-639"), taxonomy, client=None)
|
||||
assert declared.families == ("injection",) and declared.tier == "static" # a declared label wins
|
||||
mechanism = classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=None)
|
||||
assert mechanism.families == ("access_control",) and mechanism.tier == "static"
|
||||
weakness = classify_pair(_case(tmp_path, cwe="CWE-89", mechanism="source_reaches_sink"), taxonomy, client=None)
|
||||
assert weakness.families == ("injection",) and weakness.tier == "static" # the generic mechanism defers to the weakness
|
||||
nothing = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=None)
|
||||
assert nothing.families == () and nothing.tier == "none" # unknown, and no client to ask
|
||||
|
||||
|
||||
def test_tier_one_reads_a_finding_from_its_tags_and_its_rule(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_finding
|
||||
|
||||
taxonomy = load_families()
|
||||
|
||||
def finding(finding_id: str, tags: list[str]) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=finding_id,
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="medium",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=12,
|
||||
function_name="leaky",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=tags,
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
tagged = classify_finding(finding("x:app.py:1", ["family:access_control"]), taxonomy, client=None)
|
||||
assert tagged.families == ("access_control",) and tagged.tier == "static"
|
||||
obligated = classify_finding(finding("obligation:protected_read:app.py:12:x", ["obligation:protected_read"]), taxonomy, client=None)
|
||||
assert obligated.families == ("access_control",)
|
||||
ruled = classify_finding(finding("python-os-command:app.py:3", ["tool-hunter"]), taxonomy, client=None)
|
||||
assert ruled.families == ("injection",) and ruled.tier == "static" # the rule's weakness names the family
|
||||
assert classify_finding(finding("tool-hunter:app.py:3", ["tool-hunter"]), taxonomy, client=None).families == ()
|
||||
|
||||
|
||||
def test_tier_one_classifies_the_shared_handler_without_any_client(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_region
|
||||
|
||||
(tmp_path / "app.py").write_text(APP)
|
||||
taxonomy = load_families()
|
||||
leaky = classify_region(tmp_path, "app.py", function="leaky", taxonomy=taxonomy, client=None)
|
||||
assert leaky.families == ("access_control",) and leaky.tier == "static"
|
||||
(tmp_path / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n")
|
||||
injected = classify_region(tmp_path, "flow.py", function="run", taxonomy=taxonomy, client=None)
|
||||
assert "injection" in injected.families
|
||||
|
||||
|
||||
def test_tier_two_asks_the_model_only_when_static_signals_say_nothing(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_pair
|
||||
|
||||
taxonomy = load_families()
|
||||
client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["output_encoding"]}), mean_logprob=-0.05)])
|
||||
answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client)
|
||||
assert answered.families == ("output_encoding",) and answered.tier == "model" and answered.confidence == -0.05
|
||||
quiet = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}))])
|
||||
classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=quiet)
|
||||
assert quiet.calls == [] # a static answer never spends a model call
|
||||
|
||||
|
||||
def test_an_unsure_or_off_vocabulary_answer_becomes_unknown(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_pair
|
||||
|
||||
taxonomy = load_families()
|
||||
invented = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["sql_injection"]}), mean_logprob=-0.01)])
|
||||
assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=invented).families == ()
|
||||
unsure = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}), mean_logprob=-4.0)])
|
||||
hedged = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=unsure)
|
||||
assert hedged.families == () and hedged.tier == "none" and hedged.confidence == -4.0
|
||||
broken = ScriptedChatClient([ChatResponse(content="not json at all")])
|
||||
assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=broken).families == ()
|
||||
|
||||
|
||||
def test_the_model_is_asked_in_json_mode_with_the_taxonomy_and_no_thinking(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_pair
|
||||
|
||||
taxonomy = load_families()
|
||||
client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["path"]}), mean_logprob=-0.1)])
|
||||
classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client)
|
||||
call = client.calls[0]
|
||||
prompt = json.dumps(call["messages"])
|
||||
assert "json" in prompt.lower() and "access_control" in prompt and "untrusted_destination" in prompt
|
||||
assert call.get("json_object") is True and call.get("logprobs") is True and call["tools"] == []
|
||||
|
||||
|
||||
def test_multi_label_answers_are_kept_in_taxonomy_order(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import classify_pair
|
||||
|
||||
taxonomy = load_families()
|
||||
client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["access_control", "injection"]}), mean_logprob=-0.2)])
|
||||
answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client)
|
||||
assert answered.families == ("injection", "access_control") # taxonomy order, so two runs compare equal
|
||||
|
||||
|
||||
def test_a_fabricated_family_label_in_a_catalog_is_rejected_by_name(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.classify import FamilyLabelError, validate_family_labels
|
||||
|
||||
taxonomy = load_families()
|
||||
good = _case(tmp_path, family="access_control")
|
||||
validate_family_labels([good], taxonomy)
|
||||
with pytest.raises(FamilyLabelError, match="not_a_family"):
|
||||
validate_family_labels([_case(tmp_path, family="not_a_family")], taxonomy)
|
||||
@@ -1,302 +0,0 @@
|
||||
"""learning-harness task 3.9: the learning subcommands (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n"
|
||||
CATALOG = (
|
||||
'[[pair]]\nname = "{name}"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "{name}-v.py"\nfixed = "{name}-f.py"\n'
|
||||
'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "seeded"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "command injection"\npath = "app.py"\nfunction = "run"\n'
|
||||
'sink = "system"\nmechanism = "source_reaches_sink"\nfamily = "injection"\n'
|
||||
)
|
||||
|
||||
|
||||
def _detector(monkeypatch: pytest.MonkeyPatch, line: int = 6) -> None:
|
||||
"""A deterministic detector that reads the file it was given and answers differently on the two sides.
|
||||
|
||||
The shipped `scripted` stub answers twice and then goes quiet, and its finding sits at line 3, outside `run`.
|
||||
Both make every family number zero, and a scoreboard assertion against zeros passes with the detector missing
|
||||
entirely — which is what these tests used to assert."""
|
||||
from openultrasast.learning.endpoint import ChatEndpoint
|
||||
from openultrasast.tool_hunter import ChatResponse, ToolCall
|
||||
|
||||
class Detector:
|
||||
def __init__(self) -> None:
|
||||
self.turn = 0
|
||||
|
||||
def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def]
|
||||
del model, tools, options
|
||||
self.turn += 1
|
||||
if self.turn % 2 == 1:
|
||||
return ChatResponse(tool_calls=(ToolCall(id=f"c{self.turn}", name="grep_repo", arguments={"pattern": "system"}),))
|
||||
body = "\n".join(str(message.get("content") or "") for message in messages)
|
||||
if "os.system" not in body:
|
||||
return ChatResponse(content="[]") # the fixed side really is different, and the detector sees it
|
||||
return ChatResponse(content=json.dumps([{"path": "app.py", "line": line, "title": "t", "rationale": "r"}]))
|
||||
|
||||
endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False)
|
||||
monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (Detector(), endpoint))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
for name in ("train-a", "hold-a"):
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
(tmp_path / "pairs.toml").write_text(
|
||||
CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout")
|
||||
)
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
|
||||
return tmp_path # no chdir: the mechanism vocabulary is found relative to the repository root
|
||||
|
||||
|
||||
def test_classify_reports_the_corpus_and_writes_the_queue(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert main(["learning", "classify", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning")]) == 0
|
||||
payload = json.loads((workspace / "learning" / "classifier.json").read_text())
|
||||
assert payload["classified"] == 2 and payload["per_family"]["injection"] == 2
|
||||
assert payload["hierarchical_credit"] is False
|
||||
assert "classified 2" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_score_reports_family_numbers_for_a_catalog(
|
||||
workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The scripted client detects the vulnerable side and is silent on the fix, so this is a real 2/2, not a zero.
|
||||
|
||||
An all-zero scoreboard would satisfy "the family key exists"; it is what a generalist detector scoring every
|
||||
family produced before round zero ran each family's own configuration."""
|
||||
_detector(monkeypatch)
|
||||
assert main(["learning", "score", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--json"]) == 0
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
block = payload["families"]["injection"]
|
||||
assert block["taxonomy_version"] and block["scorable"] == 2
|
||||
assert block["outcomes"] == {"pair_correct": 2} and block["recall"] == 1.0 and block["youden"] == 1.0
|
||||
|
||||
|
||||
def test_baseline_writes_the_noise_floor_and_publish_reads_it(
|
||||
workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_detector(monkeypatch)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"scripted",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
floors = json.loads((workspace / "learning" / "baseline" / "scripted" / "noise-floors.json").read_text())
|
||||
assert floors["k_runs"] == 3 and floors["floors"]["injection"]["pairs"] == 2
|
||||
report = json.loads((workspace / "learning" / "baseline" / "scripted" / "report.json").read_text())
|
||||
# Measured with the injection configuration, not the generalist: `family:unknown` earns no credit against any
|
||||
# real family, so a baseline built from one shared detector reports a structural zero here.
|
||||
assert report["metrics"]["injection"]["recall"] == 1.0
|
||||
assert report["per_slice"]["vibe-py/injection"]["scorable"] == 2
|
||||
capsys.readouterr()
|
||||
roadmap = workspace / "roadmap.md"
|
||||
roadmap.write_text("# Roadmap\n\n## Overview\n\ntext\n")
|
||||
assert (
|
||||
main(
|
||||
["learning", "publish", "--out", str(workspace / "learning"), "--measurements", str(workspace / "m"), "--roadmap", str(roadmap)]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert "learning-harness:begin" in roadmap.read_text()
|
||||
assert "| injection |" in roadmap.read_text()
|
||||
|
||||
|
||||
def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace: Path) -> None:
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"scripted",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"round",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--family",
|
||||
"injection",
|
||||
"--k-runs",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
rounds = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()]
|
||||
assert len(rounds) == 1 and rounds[0]["family"] == "injection"
|
||||
# Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of
|
||||
# the enum would have hidden that the round could never propose anything at all.
|
||||
# The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different
|
||||
# worlds, and only one of them is a fixable configuration error.
|
||||
assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal")
|
||||
assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
|
||||
|
||||
|
||||
def test_a_recorded_proposal_makes_the_round_do_something(workspace: Path) -> None:
|
||||
"""`--proposals` is what makes a round runnable at all without the optional extra."""
|
||||
for command in (
|
||||
["learning", "baseline", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--k-runs", "3"],
|
||||
):
|
||||
assert main([*command, "--model", "scripted"]) == 0
|
||||
proposals = workspace / "proposals.json"
|
||||
proposals.write_text(
|
||||
json.dumps([{"hypothesis": "ask about quoting", "lever": "checklist", "change": {"checklist.md": "- ask about quoting\n"}}])
|
||||
)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"round",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--family",
|
||||
"injection",
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--proposals",
|
||||
str(proposals),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
(record,) = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()]
|
||||
assert record["reason"] != "no_proposal", "a recorded proposal reached the round"
|
||||
assert json.loads((workspace / "learning" / "rounds" / "1" / "proposal.json").read_text())["lever"] == "checklist"
|
||||
|
||||
|
||||
def test_a_missing_endpoint_is_recorded_and_the_rest_still_runs(workspace: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"none",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert "learning_endpoint_unavailable" in out
|
||||
assert (workspace / "learning" / "baseline" / "none" / "report.json").is_file() # the run still recorded what it could
|
||||
|
||||
|
||||
def test_every_subcommand_is_listed_in_help(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
main(["learning", "--help"])
|
||||
out = capsys.readouterr().out
|
||||
for name in ("classify", "score", "baseline", "round", "publish"):
|
||||
assert name in out
|
||||
|
||||
|
||||
def test_the_pairs_command_scores_k_runs_per_side(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Grepping `--help` for the flag passes with the capability entirely absent, which is what it used to be."""
|
||||
from openultrasast import pairs as pairs_module
|
||||
|
||||
calls: list[str] = []
|
||||
real = pairs_module.make_hunter_scan
|
||||
|
||||
def counting(client: object, model: str, **kwargs: object): # type: ignore[no-untyped-def]
|
||||
inner = real(client, model, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
def scan(root: Path): # type: ignore[no-untyped-def]
|
||||
calls.append(root.name)
|
||||
return inner(root)
|
||||
|
||||
return scan
|
||||
|
||||
monkeypatch.setattr(pairs_module, "make_hunter_scan", counting)
|
||||
monkeypatch.setattr("openultrasast.cli.make_hunter_scan", counting)
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_MODEL", "scripted")
|
||||
assert main(["pairs", "--catalog", str(workspace / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "4"]) == 0
|
||||
assert calls.count("vuln") == 8 and calls.count("fixed") == 8 # two pairs, four runs per side
|
||||
|
||||
|
||||
def test_a_baseline_records_what_the_endpoint_actually_spent(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Req 8.3: cost per pair is published beside every number, so the run has to read it off the client.
|
||||
|
||||
The client meters itself; nothing passed the meter to the baseline, so every published cost would have been
|
||||
0.0 — and the round's cost cap, which reads the same meter, could never have fired either (Req 9.8)."""
|
||||
from openultrasast.learning.endpoint import ChatEndpoint
|
||||
from openultrasast.tool_hunter import ChatResponse
|
||||
|
||||
class Metered:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def]
|
||||
del model, messages, tools, options
|
||||
self.calls += 1
|
||||
return ChatResponse(content="[]")
|
||||
|
||||
def cost_usd(self) -> float:
|
||||
return 0.25 * self.calls
|
||||
|
||||
client = Metered()
|
||||
endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False)
|
||||
monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (client, endpoint))
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(workspace / "pairs.toml"),
|
||||
"--out",
|
||||
str(workspace / "learning"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"metered",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
report = json.loads((workspace / "learning" / "baseline" / "metered" / "report.json").read_text())
|
||||
assert client.calls > 0
|
||||
assert report["cost_usd"] == pytest.approx(0.25 * client.calls)
|
||||
@@ -1,215 +0,0 @@
|
||||
"""Rows fixed after the detector's training cutoff are reported apart (learning-harness 4.3, Req 10.5).
|
||||
|
||||
A model that may have read the fix cannot be measured on it the same way as one that cannot have. The corpus
|
||||
carries the date at the precision the recipe recorded — a year for every row harvested so far — so a row counts as
|
||||
post-cutoff only when its *earliest possible* date is after the cutoff. Rows with no date at all are counted
|
||||
separately rather than quietly folded into either side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.scoring import PairFamilyScore, aggregate
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
|
||||
|
||||
|
||||
def _score(pair: str, outcome: str, *, fix_date: str = "") -> PairFamilyScore:
|
||||
return PairFamilyScore(pair=pair, family="injection", slice="vibe-py", runs=(outcome,), outcome=outcome, fix_date=fix_date) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_every_vendored_row_carries_the_date_its_recipe_recorded() -> None:
|
||||
cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG))
|
||||
dated = [case for case in cases if case.fix_date]
|
||||
assert len(dated) >= 250, "the generator fills the date from the recipe; almost every row has a year"
|
||||
assert all(len(case.fix_date) in {4, 10} for case in dated), "a date is a year or a full ISO day, never anything else"
|
||||
by_name = {case.name: case for case in cases}
|
||||
for slice_name in ("vibe-py", "agent-vfc"):
|
||||
for recipe in tomllib.loads((Path("benchmarks/pairs") / slice_name / "recipes.toml").read_text())["recipe"]:
|
||||
case = by_name.get(str(recipe["name"]))
|
||||
if case is not None and recipe.get("year"):
|
||||
assert case.fix_date.startswith(str(recipe["year"]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fix_date", "cutoff", "expected"),
|
||||
[
|
||||
("2026", "2025-07-01", True), # every day of 2026 is after the cutoff
|
||||
("2026", "2026-03-01", False), # a year-precision row that might be before it is not claimed
|
||||
("2026-04-02", "2026-03-01", True),
|
||||
("2024", "2025-07-01", False),
|
||||
("2026-05", "2026-03-01", True), # month precision: the earliest day of May is after 1 March
|
||||
("2026-05", "2026-06-01", False), # and before 1 June, so the row is not claimed
|
||||
("2025", "2025", False), # a cutoff coarser than a day is refused rather than compared
|
||||
("", "2025-07-01", False),
|
||||
],
|
||||
)
|
||||
def test_a_row_is_post_cutoff_only_when_its_earliest_possible_date_is_after_it(fix_date: str, cutoff: str, expected: bool) -> None:
|
||||
from openultrasast.learning.scoring import is_post_cutoff
|
||||
|
||||
assert is_post_cutoff(fix_date, cutoff) is expected
|
||||
|
||||
|
||||
def test_the_scoreboard_reports_the_post_cutoff_slice_the_cutoff_and_the_undated_count() -> None:
|
||||
scores = [
|
||||
_score("new-correct", "pair_correct", fix_date="2026"),
|
||||
_score("new-missed", "both_silent", fix_date="2026"),
|
||||
_score("old-correct", "pair_correct", fix_date="2021"),
|
||||
_score("undated", "pair_correct"),
|
||||
]
|
||||
block = aggregate(scores, taxonomy=load_families(), cutoff="2025-07-01")["injection"]
|
||||
assert block.cutoff == "2025-07-01" and block.undated == 1
|
||||
assert block.post_cutoff["scorable"] == 2 and block.post_cutoff["recall"] == 0.5
|
||||
assert block.scorable == 4, "the slice is reported beside the whole number, never instead of it"
|
||||
payload = block.to_dict()
|
||||
json.dumps(payload)
|
||||
assert payload["cutoff"] == "2025-07-01" and payload["post_cutoff"]["scorable"] == 2
|
||||
|
||||
|
||||
def test_without_a_configured_cutoff_the_scoreboard_says_so_rather_than_inventing_one() -> None:
|
||||
block = aggregate([_score("a", "pair_correct", fix_date="2026")], taxonomy=load_families())["injection"]
|
||||
assert block.cutoff == "" and block.post_cutoff == {} and block.undated == 0
|
||||
|
||||
|
||||
def test_the_score_carries_the_date_from_the_pair(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
(tmp_path / "v.py").write_text("def run():\n return 1\n")
|
||||
case = PairCase(
|
||||
name="p",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "v.py",
|
||||
relpath="app.py",
|
||||
expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
fix_date="2026",
|
||||
)
|
||||
score = score_pair_family(case, "injection", [[]], [[]], ranges={"app.py": (("run", 1, 2),)}, taxonomy=load_families())
|
||||
assert score.fix_date == "2026"
|
||||
|
||||
|
||||
def test_the_published_table_names_the_post_cutoff_slice(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
directory = out / "baseline" / "m"
|
||||
directory.mkdir(parents=True)
|
||||
(directory / "report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model": "m",
|
||||
"k_runs": 5,
|
||||
"cost_usd": 1.0,
|
||||
"taxonomy_version": "1",
|
||||
"metrics": {
|
||||
"injection": {
|
||||
"family": "injection",
|
||||
"scorable": 4,
|
||||
"unscorable": {},
|
||||
"recall": 0.5,
|
||||
"youden": 0.25,
|
||||
"hierarchical_credit": False,
|
||||
"cutoff": "2025-07-01",
|
||||
"undated": 1,
|
||||
"post_cutoff": {"scorable": 2, "recall": 0.5, "youden": 0.5},
|
||||
}
|
||||
},
|
||||
"floors": {},
|
||||
}
|
||||
)
|
||||
)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text("# Roadmap\n")
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert "2025-07-01" in report.table
|
||||
assert "post-cutoff" in report.table.lower()
|
||||
assert "undated" in report.table.lower()
|
||||
|
||||
|
||||
def test_the_command_line_reports_the_slice_the_cutoff_and_the_undated_count(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Req 10.5 says every scoreboard, which includes the one an operator actually reads."""
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
for name in ("train-a", "hold-a"):
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
catalog = CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout")
|
||||
(tmp_path / "pairs.toml").write_text(catalog.replace('review_tier = "seeded"', 'review_tier = "seeded"\nfix_date = "2026"'))
|
||||
(tmp_path / "openultrasast.toml").write_text('[learning]\nenabled = true\ncutoff_date = "2025-07-01"\n')
|
||||
_detector(monkeypatch)
|
||||
captured: list[str] = []
|
||||
monkeypatch.setattr("builtins.print", lambda *args, **kwargs: captured.append(" ".join(str(item) for item in args)))
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"score",
|
||||
"--catalog",
|
||||
str(tmp_path / "pairs.toml"),
|
||||
"--out",
|
||||
str(tmp_path / "learning"),
|
||||
"--config",
|
||||
str(tmp_path / "openultrasast.toml"),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
line = next(item for item in captured if item.startswith("learning score injection"))
|
||||
assert "post-cutoff(2025-07-01)=2" in line and "undated=0" in line
|
||||
|
||||
|
||||
def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`--slice vfc` selected the vfc rows and then `run_baseline` filtered them all out again.
|
||||
|
||||
Its `slices` default is the two web slices, so a baseline over the memory-safety slice — the run Req 8.4 asks
|
||||
for as a measured, non-gating number — produced an empty report and said nothing about it."""
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
(tmp_path / "a-v.py").write_text(VULN)
|
||||
(tmp_path / "a-f.py").write_text(FIXED)
|
||||
(tmp_path / "pairs.toml").write_text(CATALOG.format(name="a", split="train").replace('slice = "vibe-py"', 'slice = "vfc"'))
|
||||
_detector(monkeypatch)
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"learning",
|
||||
"baseline",
|
||||
"--catalog",
|
||||
str(tmp_path / "pairs.toml"),
|
||||
"--slice",
|
||||
"vfc",
|
||||
"--out",
|
||||
str(tmp_path / "l"),
|
||||
"--k-runs",
|
||||
"3",
|
||||
"--model",
|
||||
"m",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
# A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run
|
||||
# over the web families.
|
||||
report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text())
|
||||
assert report["slices"] == ["vfc"]
|
||||
assert report["metrics"]["injection"]["scorable"] == 1
|
||||
@@ -1,167 +0,0 @@
|
||||
"""learning-harness task 2.5: per-family detector configurations and the family runner (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall
|
||||
|
||||
APP = (
|
||||
"from flask import request\n\n\n"
|
||||
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
|
||||
" return Book.query.filter_by(book_title=title).first()\n\n\n"
|
||||
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
|
||||
" owner = request.args.get('owner')\n"
|
||||
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n"
|
||||
)
|
||||
REPLY = '[{"path": "app.py", "line": 12, "title": "unconstrained read", "rationale": "no owner check"}]'
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path: Path) -> Path:
|
||||
(tmp_path / "app.py").write_text(APP)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _write_config(root: Path, family: str, **overrides: object) -> Path:
|
||||
directory = root / family
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "prompt.md").write_text(str(overrides.get("prompt", f"You hunt {family} only.")))
|
||||
(directory / "checklist.md").write_text(str(overrides.get("checklist", "- did the handler check ownership?")))
|
||||
tools = overrides.get("tools", ["read_definition", "obligations"])
|
||||
(directory / "family.toml").write_text(
|
||||
f'version = "{overrides.get("version", "1")}"\ntools = {json.dumps(tools)}\n'
|
||||
f"max_steps = {overrides.get('max_steps', 3)}\nmax_cost_usd = {overrides.get('max_cost_usd', 0.5)}\n"
|
||||
f"max_chars = {overrides.get('max_chars', 4000)}\n"
|
||||
)
|
||||
(directory / "hard_negatives.jsonl").write_text(json.dumps({"note": "a guarded handler is not a finding"}) + "\n")
|
||||
(directory / "counterexamples.jsonl").write_text("")
|
||||
return directory
|
||||
|
||||
|
||||
def test_a_configuration_loads_every_part_it_needs(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import load_family_configs
|
||||
|
||||
_write_config(tmp_path, "access_control")
|
||||
configs = load_family_configs(tmp_path, load_families())
|
||||
config = configs["access_control"]
|
||||
assert config.family == "access_control" and config.version == "1"
|
||||
assert config.prompt.startswith("You hunt access_control") and "ownership" in config.checklist
|
||||
assert config.tools == ("read_definition", "obligations")
|
||||
assert config.max_steps == 3 and config.max_cost_usd == 0.5 and config.max_chars == 4000
|
||||
assert config.hard_negatives[0]["note"].startswith("a guarded handler")
|
||||
assert config.counterexamples == ()
|
||||
assert load_family_configs(tmp_path, load_families())["access_control"] == config # loading twice is the same config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "needle"),
|
||||
[
|
||||
({"prompt": "x" * 9000}, "length"),
|
||||
({"tools": ["read_file", "run_shell"]}, "run_shell"),
|
||||
({"version": ""}, "version"),
|
||||
],
|
||||
ids=["over_cap_prompt", "unknown_tool", "no_version"],
|
||||
)
|
||||
def test_the_loader_refuses_a_broken_configuration_by_name(tmp_path: Path, overrides: dict[str, object], needle: str) -> None:
|
||||
from openultrasast.learning.detectors import DetectorConfigError, load_family_configs
|
||||
|
||||
_write_config(tmp_path, "access_control", **overrides)
|
||||
with pytest.raises(DetectorConfigError, match=needle):
|
||||
load_family_configs(tmp_path, load_families())
|
||||
|
||||
|
||||
def test_a_directory_that_is_not_a_family_is_refused_by_name(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import DetectorConfigError, load_family_configs
|
||||
|
||||
_write_config(tmp_path, "access_control")
|
||||
(tmp_path / "sql_injection").mkdir()
|
||||
(tmp_path / "sql_injection" / "prompt.md").write_text("x")
|
||||
with pytest.raises(DetectorConfigError, match="sql_injection"):
|
||||
load_family_configs(tmp_path, load_families())
|
||||
|
||||
|
||||
def test_the_runner_uses_the_family_prompt_tools_and_tags(repo: Path, tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector
|
||||
|
||||
configs_dir = tmp_path / "configs"
|
||||
_write_config(configs_dir, "access_control")
|
||||
config = load_family_configs(configs_dir, load_families())["access_control"]
|
||||
client = ScriptedChatClient(
|
||||
[
|
||||
ChatResponse(tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),)),
|
||||
ChatResponse(content=REPLY),
|
||||
]
|
||||
)
|
||||
findings = run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m")
|
||||
assert [finding.tags for finding in findings] == [["family:access_control", "detector:access_control@1"]]
|
||||
assert findings[0].evidence_level == "suspicion"
|
||||
call = client.calls[0]
|
||||
assert call["messages"][0]["content"] == "You hunt access_control only."
|
||||
assert [tool["function"]["name"] for tool in call["tools"]] == ["read_definition", "obligations"]
|
||||
user = str(call["messages"][1]["content"])
|
||||
assert "def leaky" in user and "def guarded" in user # the whole file, not an excerpt around a hotspot
|
||||
assert "ownership" in user and "a guarded handler is not a finding" in user
|
||||
|
||||
|
||||
def test_a_region_runs_every_admitted_family_plus_the_generalist(repo: Path, tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import Region, load_family_configs, run_region_detectors
|
||||
|
||||
configs_dir = tmp_path / "configs"
|
||||
for family in ("access_control", "injection", "unknown"):
|
||||
_write_config(configs_dir, family, tools=["read_file"])
|
||||
configs = load_family_configs(configs_dir, load_families())
|
||||
client = ScriptedChatClient(
|
||||
[
|
||||
ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)),
|
||||
ChatResponse(content=REPLY),
|
||||
]
|
||||
* 4
|
||||
)
|
||||
region = Region(path="app.py", function="leaky", families=("access_control",))
|
||||
findings = run_region_detectors(repo, region, configs, client=client, model="m")
|
||||
families = sorted({tag for finding in findings for tag in finding.tags if tag.startswith("family:")})
|
||||
assert families == ["family:access_control", "family:unknown"] # the admitted family and the generalist, not injection
|
||||
assert len(client.calls) == 4 # two turns each, and nothing else was run
|
||||
|
||||
|
||||
def test_the_runner_never_offers_a_tool_outside_the_curated_set(repo: Path, tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector
|
||||
from openultrasast.tool_hunter import HUNTER_TOOLS
|
||||
|
||||
configs_dir = tmp_path / "configs"
|
||||
_write_config(
|
||||
configs_dir, "injection", tools=["read_file", "grep_repo", "find_refs", "read_definition", "entry_points", "flows", "obligations"]
|
||||
)
|
||||
config = load_family_configs(configs_dir, load_families())["injection"]
|
||||
client = ScriptedChatClient([ChatResponse(content="")])
|
||||
run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m")
|
||||
offered = {tool["function"]["name"] for tool in client.calls[0]["tools"]}
|
||||
assert offered == {str(tool["function"]["name"]) for tool in HUNTER_TOOLS}
|
||||
assert "bash" not in offered and "run_shell" not in offered
|
||||
|
||||
|
||||
def test_round_zero_can_write_a_configuration_for_every_family(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import load_family_configs, write_default_configs
|
||||
|
||||
taxonomy = load_families()
|
||||
written = write_default_configs(tmp_path, taxonomy, prompt="You are a security hunter.", version="0")
|
||||
assert set(written) == {family.id for family in taxonomy.families}
|
||||
configs = load_family_configs(tmp_path, taxonomy)
|
||||
assert set(configs) == {family.id for family in taxonomy.families}
|
||||
assert all(config.version == "0" for config in configs.values())
|
||||
assert all(config.prompt.startswith("You are a security hunter.") for config in configs.values())
|
||||
assert configs["access_control"].prompt != configs["injection"].prompt # each carries its own family's definition
|
||||
|
||||
|
||||
def test_a_configuration_with_no_tools_is_refused_rather_than_reporting_nothing(tmp_path: Path) -> None:
|
||||
"""A detector offered no tools never calls one, and the hunter loop then returns nothing at all."""
|
||||
from openultrasast.learning.detectors import DetectorConfigError, load_family_configs
|
||||
|
||||
_write_config(tmp_path, "injection", tools=[])
|
||||
with pytest.raises(DetectorConfigError, match="tools"):
|
||||
load_family_configs(tmp_path, load_families())
|
||||
@@ -1,112 +0,0 @@
|
||||
"""learning-harness task 2.8: the round journal, the archive and the rejected buffer (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.learning.scoring import PairFamilyScore
|
||||
|
||||
|
||||
def _record(round_number: int, family: str, outcome: str, **extra: object) -> object:
|
||||
from openultrasast.learning.journal import Attribution, RoundRecord
|
||||
|
||||
defaults: dict[str, object] = {
|
||||
"round": round_number,
|
||||
"family": family,
|
||||
"hypothesis": "the checklist never asks about ownership",
|
||||
"levers": ("checklist",),
|
||||
"predicted_affected": (family,),
|
||||
"predicted_at_risk": ("injection",),
|
||||
"outcome": outcome,
|
||||
"reason": "",
|
||||
"target_train_delta": 0.1,
|
||||
"target_holdout_delta": 0.2,
|
||||
"sweep": {"injection": 0},
|
||||
"attribution": Attribution(flipped_predicted=2, flipped_unpredicted=0, precision=1.0),
|
||||
"cost_usd": 0.42,
|
||||
"taxonomy_version": "1",
|
||||
"config_version": "2",
|
||||
}
|
||||
return RoundRecord(**{**defaults, **extra}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_record_round_trips_through_the_file(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import LearningJournal
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
assert journal.rounds() == ()
|
||||
record = _record(1, "access_control", "accepted")
|
||||
journal.append(record)
|
||||
journal.append(_record(2, "injection", "rejected", reason="tie"))
|
||||
rounds = journal.rounds()
|
||||
assert [item.round for item in rounds] == [1, 2] and rounds[0] == record
|
||||
assert rounds[0].attribution.precision == 1.0 and rounds[0].sweep == {"injection": 0}
|
||||
assert rounds[1].outcome == "rejected" and rounds[1].reason == "tie"
|
||||
rows = [json.loads(line) for line in (tmp_path / "journal.jsonl").read_text().splitlines() if line.strip()]
|
||||
assert len(rows) == 2 and rows[0]["hypothesis"].startswith("the checklist")
|
||||
|
||||
|
||||
def test_the_journal_is_append_only_and_a_round_number_is_never_reused(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import JournalError, LearningJournal
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
journal.append(_record(1, "access_control", "accepted"))
|
||||
before = (tmp_path / "journal.jsonl").read_text()
|
||||
try:
|
||||
journal.append(_record(1, "injection", "accepted"))
|
||||
raise AssertionError("a repeated round number must be refused")
|
||||
except JournalError as error:
|
||||
assert "1" in str(error)
|
||||
assert (tmp_path / "journal.jsonl").read_text() == before
|
||||
assert journal.next_round() == 2
|
||||
|
||||
|
||||
def test_the_rejected_buffer_returns_only_the_target_family(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import LearningJournal
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
journal.append(_record(1, "access_control", "rejected", hypothesis="ask about ownership", reason="tie"))
|
||||
journal.append(_record(2, "injection", "rejected", hypothesis="ask about quoting", reason="tie"))
|
||||
journal.append(_record(3, "access_control", "accepted", hypothesis="give it the obligations tool"))
|
||||
assert journal.rejected_buffer("access_control") == ("ask about ownership",)
|
||||
assert journal.rejected_buffer("injection") == ("ask about quoting",)
|
||||
assert journal.rejected_buffer("memory") == ()
|
||||
|
||||
|
||||
def test_the_archive_names_one_winning_configuration_per_pair_and_family(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import Archive
|
||||
|
||||
archive = Archive(tmp_path / "archive.jsonl")
|
||||
correct = PairFamilyScore(pair="a", family="access_control", runs=("pair_correct",), outcome="pair_correct")
|
||||
missed = PairFamilyScore(pair="a", family="access_control", runs=("both_silent",), outcome="both_silent")
|
||||
archive.record("access_control", "1", correct)
|
||||
archive.record("access_control", "2", missed) # a later version that loses this pair does not take it
|
||||
archive.record(
|
||||
"access_control", "2", PairFamilyScore(pair="b", family="access_control", runs=("pair_correct",), outcome="pair_correct")
|
||||
)
|
||||
assert archive.winners("access_control") == {"a": "1", "b": "2"}
|
||||
assert archive.winners("injection") == {}
|
||||
reopened = Archive(tmp_path / "archive.jsonl")
|
||||
assert reopened.winners("access_control") == {"a": "1", "b": "2"} # survives a reload, like the store it copies
|
||||
|
||||
|
||||
def test_a_later_version_takes_a_pair_only_by_winning_it(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import Archive
|
||||
|
||||
archive = Archive(tmp_path / "archive.jsonl")
|
||||
archive.record("injection", "1", PairFamilyScore(pair="a", family="injection", runs=("both_silent",), outcome="both_silent"))
|
||||
assert archive.winners("injection") == {}
|
||||
archive.record("injection", "2", PairFamilyScore(pair="a", family="injection", runs=("pair_correct",), outcome="pair_correct"))
|
||||
assert archive.winners("injection") == {"a": "2"}
|
||||
|
||||
|
||||
def test_everything_persisted_passes_redaction(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.journal import LearningJournal
|
||||
|
||||
leaked = "api_key = " + "'" + "A" * 24 + "'" # assembled here so no key-shaped literal sits in the file
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
journal.append(_record(1, "access_control", "rejected", hypothesis=f"the prompt should show {leaked}", reason="tie"))
|
||||
text = (tmp_path / "journal.jsonl").read_text()
|
||||
assert "A" * 24 not in text and "REDACTED" in text
|
||||
assert journal.rounds()[0].hypothesis.endswith("'")
|
||||
@@ -1,151 +0,0 @@
|
||||
"""What the pairs hunter path must prove (learning-harness review round 3, Req 3.2, 3.5).
|
||||
|
||||
Three defects, one test each: K was declared and never threaded, so every stage scored one run; the fixed
|
||||
side was judged against the *vulnerable* side's function ranges, which differ on most of the corpus; and
|
||||
the generalist hunter emitted findings with no family tag, which the class-aware scorer can never credit,
|
||||
so `ousast pairs --hunter` reported recall zero by construction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import PairCase, evaluate_catalog
|
||||
|
||||
# The fix inserts two guard lines above the handler, so `run` starts on a different line on each side.
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = (
|
||||
"from flask import request\nimport shlex\nimport os\n\n\n"
|
||||
"def _safe(value):\n return shlex.quote(value or '')\n\n\n"
|
||||
"def run():\n os.system(_safe(request.args.get('cmd')))\n"
|
||||
)
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str = "p") -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78",
|
||||
vulnerability_class="x",
|
||||
path="app.py",
|
||||
evidence="",
|
||||
function="run",
|
||||
mechanism="other",
|
||||
family="injection",
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
review_tier="seeded",
|
||||
)
|
||||
|
||||
|
||||
def _finding(line: int, *, family: str | None = "injection") -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"tool-hunter:app.py:{line}",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["tool-hunter", *([f"family:{family}"] if family else [])],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def test_the_k_runs_flag_reaches_the_detector(tmp_path: Path) -> None:
|
||||
"""Req 3.5: three runs per side, per pair. A parameter no caller can set is not a K-run floor."""
|
||||
calls: list[str] = []
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
calls.append(root.name)
|
||||
return [_finding(6)] if root.name == "vuln" else []
|
||||
|
||||
evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
|
||||
assert calls.count("vuln") == 3 and calls.count("fixed") == 3
|
||||
|
||||
|
||||
def test_fewer_than_three_runs_is_refused(tmp_path: Path) -> None:
|
||||
with pytest.raises(ValueError, match="three runs"):
|
||||
evaluate_catalog([_case(tmp_path)], hunter=lambda root: [], k_runs=1)
|
||||
|
||||
|
||||
def test_the_fixed_side_is_judged_against_the_fixed_side_line_numbers(tmp_path: Path) -> None:
|
||||
"""`run` sits at line 5 on the vulnerable side and line 10 on the fixed side.
|
||||
|
||||
Scoring the fixed side against the vulnerable side's span reads a finding inside the fixed handler as
|
||||
outside it, so a detector that flags both sides is recorded as silent on the fix — a leak counted as a
|
||||
clean pass, and always in the direction that flatters the number."""
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
return [_finding(6)] if root.name == "vuln" else [_finding(11)]
|
||||
|
||||
result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
|
||||
hunter = result.scorers["vibe-py"]["hunter"]
|
||||
assert hunter.detected_vuln == 1
|
||||
assert hunter.silent_fix == 0, "the finding is inside `run` on the fixed side; that is a leak"
|
||||
assert hunter.pair_correct == 0
|
||||
|
||||
|
||||
def test_a_generalist_finding_without_a_family_is_never_a_detection(tmp_path: Path) -> None:
|
||||
"""The other half of the same defect: `pairs --hunter` used to feed the scorer untagged findings."""
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
return [_finding(6, family=None)] if root.name == "vuln" else []
|
||||
|
||||
result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
|
||||
assert result.scorers["vibe-py"]["hunter"].detected_vuln == 0
|
||||
|
||||
|
||||
def test_the_hunter_reports_the_family_it_believes_and_that_is_what_is_credited(tmp_path: Path) -> None:
|
||||
"""A generalist that must name the family is answerable; one that names nothing cannot be scored at all."""
|
||||
from openultrasast.tool_hunter import _findings_from_content
|
||||
|
||||
findings = _findings_from_content(
|
||||
tmp_path,
|
||||
[],
|
||||
'[{"path": "app.py", "line": 6, "family": "injection", "title": "t", "rationale": "r"}]',
|
||||
)
|
||||
assert findings and "family:injection" in findings[0].tags
|
||||
untagged = _findings_from_content(tmp_path, [], '[{"path": "app.py", "line": 6, "title": "t"}]')
|
||||
assert untagged and not [tag for tag in untagged[0].tags if tag.startswith("family:")]
|
||||
|
||||
|
||||
def test_the_pair_hunter_asks_for_a_family_from_the_closed_list(tmp_path: Path) -> None:
|
||||
"""The prompt has to name the taxonomy, or the model invents family names and every finding is noise."""
|
||||
from openultrasast.pairs import make_hunter_scan
|
||||
|
||||
seen: list[str] = []
|
||||
|
||||
class Client:
|
||||
def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def]
|
||||
seen.append("\n".join(str(message.get("content") or "") for message in messages))
|
||||
from openultrasast.tool_hunter import ChatResponse
|
||||
|
||||
return ChatResponse(content="[]", tool_calls=())
|
||||
|
||||
(tmp_path / "app.py").write_text(VULN)
|
||||
make_hunter_scan(Client(), "m", taxonomy=load_families())(tmp_path)
|
||||
prompt = "\n".join(seen)
|
||||
assert "injection" in prompt and "access_control" in prompt and "untrusted_destination" in prompt
|
||||
assert '"family"' in prompt
|
||||
@@ -1,110 +0,0 @@
|
||||
"""learning-harness task 3.2: family scoring on the pairs hunter path (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.pairs import PairCase, evaluate_pair
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', request.args.get('cmd')], check=False)\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, *, family: str | None = "injection", unscorable: str | None = None) -> PairCase:
|
||||
(tmp_path / "v.py").write_text(VULN)
|
||||
(tmp_path / "f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name="p",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
unscorable=unscorable,
|
||||
)
|
||||
|
||||
|
||||
def _finding(family: str, line: int) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"detector:app.py:{line}",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", "detector:injection@1"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _hunter(vuln: list[StaticFinding], fixed: list[StaticFinding]): # type: ignore[no-untyped-def]
|
||||
seen: list[Path] = []
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
seen.append(root)
|
||||
return vuln if root.name == "vuln" else fixed
|
||||
|
||||
scan.seen = seen # type: ignore[attr-defined]
|
||||
return scan
|
||||
|
||||
|
||||
def test_the_hunter_path_scores_by_family_and_function_not_by_text(tmp_path: Path) -> None:
|
||||
right = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], []))
|
||||
assert right.detected_vuln and right.silent_fix and right.pair_correct
|
||||
assert right.detection_kinds == ("hunter",) and right.family == "injection"
|
||||
wrong_family = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("access_control", 6)], []))
|
||||
assert not wrong_family.detected_vuln # the right place, the wrong family
|
||||
loud = StaticFinding(**{**_finding("access_control", 6).__dict__, "title": "CWE-78 command injection in run"})
|
||||
assert not evaluate_pair(_case(tmp_path), hunter=_hunter([loud], [])).detected_vuln # the words never count
|
||||
|
||||
|
||||
def test_a_finding_of_another_family_on_the_fixed_side_is_not_a_leak(tmp_path: Path) -> None:
|
||||
outcome = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("access_control", 6)]))
|
||||
assert outcome.pair_correct and outcome.silent_fix
|
||||
assert outcome.fix_findings == 1 and outcome.fix_leaks == 0 # counted, never charged
|
||||
leaked = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("injection", 6)]))
|
||||
assert leaked.detected_vuln and not leaked.silent_fix and leaked.fix_leaks == 1
|
||||
|
||||
|
||||
def test_an_unscorable_pair_is_reported_as_such_and_never_run(tmp_path: Path) -> None:
|
||||
hunter = _hunter([_finding("injection", 6)], [])
|
||||
outcome = evaluate_pair(_case(tmp_path, unscorable="identical_twin"), hunter=hunter)
|
||||
assert outcome.unscorable == "identical_twin" and not outcome.detected_vuln
|
||||
assert hunter.seen == [] # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_a_pair_with_no_family_label_falls_back_to_the_classifier(tmp_path: Path) -> None:
|
||||
outcome = evaluate_pair(_case(tmp_path, family=None), hunter=_hunter([_finding("injection", 6)], []))
|
||||
assert outcome.family == "injection" and outcome.pair_correct # CWE-78 classifies as injection
|
||||
|
||||
|
||||
def test_the_overlay_path_is_untouched_by_the_family_scorer(tmp_path: Path) -> None:
|
||||
overlay = evaluate_pair(_case(tmp_path))
|
||||
assert overlay.detection_kinds != ("hunter",) and overlay.family == ""
|
||||
|
||||
|
||||
def test_the_scoreboard_carries_the_family_block(tmp_path: Path) -> None:
|
||||
from openultrasast.pairs import evaluate_catalog, result_payload
|
||||
|
||||
result = evaluate_catalog([_case(tmp_path)], hunter=_hunter([_finding("injection", 6)], []), inventory_sidecar=False)
|
||||
payload = result_payload(result)
|
||||
assert "per_family" in payload and payload["per_family"]["injection"]["scorable"] == 1
|
||||
assert payload["per_family"]["injection"]["recall"] == 1.0
|
||||
assert payload["per_family"]["injection"]["hierarchical_credit"] is False
|
||||
assert payload["per_family"]["injection"]["taxonomy_version"]
|
||||
@@ -1,222 +0,0 @@
|
||||
"""learning-harness task 3.5: failure facts, the scripted proposer and the HarnessX proposer (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learning.detectors import FamilyConfig
|
||||
from openultrasast.learning.journal import LearningJournal, RoundRecord
|
||||
from openultrasast.learning.scoring import PairFamilyScore
|
||||
|
||||
|
||||
def _config(family: str = "access_control") -> FamilyConfig:
|
||||
return FamilyConfig(
|
||||
family=family,
|
||||
version="0",
|
||||
prompt="You hunt access control.",
|
||||
checklist="- did the handler check ownership?",
|
||||
tools=("read_definition", "obligations"),
|
||||
max_steps=3,
|
||||
max_cost_usd=0.5,
|
||||
max_chars=4000,
|
||||
)
|
||||
|
||||
|
||||
def _score(pair: str, outcome: str, *, family: str = "access_control") -> PairFamilyScore:
|
||||
return PairFamilyScore(pair=pair, family=family, runs=(outcome,), outcome=outcome, slice="vibe-py") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_failure_facts_carry_the_misses_the_leaks_and_the_rejected_buffer(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import build_failure_facts
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
journal.append(
|
||||
RoundRecord(
|
||||
round=1,
|
||||
family="access_control",
|
||||
hypothesis="ask about ownership",
|
||||
levers=("checklist",),
|
||||
predicted_affected=("access_control",),
|
||||
predicted_at_risk=(),
|
||||
outcome="rejected",
|
||||
reason="tie",
|
||||
)
|
||||
)
|
||||
scores = [_score("miss-one", "both_silent"), _score("leak-one", "both_flagged"), _score("good", "pair_correct")]
|
||||
facts = build_failure_facts(
|
||||
scores, family="access_control", config=_config(), journal=journal, train_pairs={"miss-one", "leak-one", "good"}
|
||||
)
|
||||
assert [item.pair for item in facts.misses] == ["miss-one"]
|
||||
assert [item.pair for item in facts.leaks] == ["leak-one"]
|
||||
assert facts.rejected == ("ask about ownership",)
|
||||
assert facts.config.family == "access_control" and facts.family == "access_control"
|
||||
|
||||
|
||||
def test_the_proposer_is_never_handed_a_holdout_pair(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import build_failure_facts
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
scores = [_score("train-miss", "both_silent"), _score("holdout-miss", "both_silent")]
|
||||
facts = build_failure_facts(scores, family="access_control", config=_config(), journal=journal, train_pairs={"train-miss"})
|
||||
assert [item.pair for item in facts.misses] == ["train-miss"]
|
||||
assert "holdout-miss" not in json.dumps(facts.to_dict())
|
||||
|
||||
|
||||
def test_the_scripted_proposer_returns_one_change_for_one_family(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import Proposal, ScriptedProposer, build_failure_facts
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"})
|
||||
scripted = ScriptedProposer(
|
||||
[
|
||||
Proposal(
|
||||
hypothesis="h",
|
||||
lever="checklist",
|
||||
change={"checklist.md": "- ask about ownership\n"},
|
||||
predicted_affected=("access_control",),
|
||||
)
|
||||
]
|
||||
)
|
||||
proposal = scripted.propose(facts)
|
||||
assert proposal is not None and proposal.lever == "checklist" and set(proposal.change) == {"checklist.md"}
|
||||
assert scripted.propose(facts) is None # a scripted proposer runs out rather than repeating itself
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "lever", "needle"),
|
||||
[
|
||||
({"../../etc/passwd": "x"}, "checklist", "outside"),
|
||||
({"family.toml": "x", "prompt.md": "y"}, "prompt", "one file"),
|
||||
({"secrets.md": "x"}, "checklist", "secrets.md"),
|
||||
({"prompt.md": "x" * 9000}, "prompt", "cap"),
|
||||
],
|
||||
ids=["escapes_the_family_directory", "two_files", "unknown_file", "over_the_cap"],
|
||||
)
|
||||
def test_a_proposal_that_reaches_beyond_its_family_is_refused_by_name(change: dict[str, str], lever: str, needle: str) -> None:
|
||||
from openultrasast.learning.proposer import Proposal, refuse_proposal
|
||||
|
||||
refusal = refuse_proposal(Proposal(hypothesis="h", lever=lever, change=change), _config()) # type: ignore[arg-type]
|
||||
assert refusal is not None and needle in refusal
|
||||
|
||||
|
||||
def test_a_well_formed_proposal_is_accepted_and_applies_inside_its_own_directory(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import Proposal, apply_proposal, refuse_proposal
|
||||
|
||||
directory = tmp_path / "access_control"
|
||||
directory.mkdir()
|
||||
(directory / "checklist.md").write_text("- old\n")
|
||||
proposal = Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}, predicted_affected=("access_control",))
|
||||
assert refuse_proposal(proposal, _config()) is None
|
||||
apply_proposal(proposal, directory)
|
||||
assert (directory / "checklist.md").read_text() == "- new\n"
|
||||
|
||||
|
||||
def test_the_write_roots_are_one_family_directory_and_exclude_the_checker_and_the_corpus(tmp_path: Path) -> None:
|
||||
from openultrasast.learning import canaries
|
||||
from openultrasast.learning.proposer import write_roots
|
||||
|
||||
roots = write_roots(tmp_path / "configs", "access_control")
|
||||
assert roots == (tmp_path / "configs" / "access_control",)
|
||||
verifier_module = Path(canaries.__file__).resolve()
|
||||
assert not any(verifier_module.is_relative_to(root.resolve()) for root in roots)
|
||||
corpus = Path("benchmarks").resolve()
|
||||
assert not any(str(corpus).startswith(str(root.resolve())) for root in roots)
|
||||
|
||||
|
||||
def test_the_harnessx_proposer_degrades_with_a_reason_when_the_extra_is_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning import proposer as proposer_module
|
||||
from openultrasast.learning.proposer import HarnessXProposer, build_failure_facts
|
||||
|
||||
journal = LearningJournal(tmp_path / "journal.jsonl")
|
||||
facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"})
|
||||
monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False)
|
||||
agent = HarnessXProposer(configs_dir=tmp_path / "configs", model="m")
|
||||
assert agent.propose(facts) is None
|
||||
assert agent.reason == "harnessx_unavailable"
|
||||
assert agent.write_roots("access_control") == (tmp_path / "configs" / "access_control",)
|
||||
|
||||
|
||||
# --- the meta-agent proposer (review round 3, task 3.5 finding 1) ------------
|
||||
|
||||
|
||||
def test_the_meta_agent_proposer_reads_its_change_out_of_a_scratch_copy(tmp_path: Path) -> None:
|
||||
"""The meta-agent never writes into the live family directory.
|
||||
|
||||
A proposer that edited the tree directly would have changed the configuration before the round snapshotted
|
||||
it, so a rejection could not put it back. It gets a copy, and the proposal is the diff of that copy."""
|
||||
from openultrasast.learning.proposer import HarnessXProposer
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
(configs / "injection").mkdir(parents=True)
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
(configs / "injection" / "family.toml").write_text('family = "injection"\nversion = "0"\n')
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def]
|
||||
seen["workspace"] = workspace
|
||||
seen["family"] = facts.family
|
||||
(workspace / "checklist.md").write_text("- ask about quoting\n")
|
||||
return "ask about quoting"
|
||||
|
||||
proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
|
||||
proposal = proposer.propose(_facts(tmp_path, "injection"))
|
||||
assert proposal is not None
|
||||
assert proposal.lever == "checklist" and proposal.change == {"checklist.md": "- ask about quoting\n"}
|
||||
assert proposal.hypothesis == "ask about quoting"
|
||||
assert seen["workspace"] != configs / "injection"
|
||||
assert (configs / "injection" / "checklist.md").read_text() == "- old\n", "the live tree was edited"
|
||||
|
||||
|
||||
def test_the_meta_agent_proposer_refuses_more_than_one_changed_file(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import HarnessXProposer
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
(configs / "injection").mkdir(parents=True)
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
(configs / "injection" / "prompt.md").write_text("p\n")
|
||||
|
||||
def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def]
|
||||
(workspace / "checklist.md").write_text("- a\n")
|
||||
(workspace / "prompt.md").write_text("- b\n")
|
||||
return "two at once"
|
||||
|
||||
proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
|
||||
assert proposer.propose(_facts(tmp_path, "injection")) is None
|
||||
assert "one file" in proposer.reason
|
||||
|
||||
|
||||
def test_the_meta_agent_proposer_refuses_a_file_no_lever_owns(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import HarnessXProposer
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
(configs / "injection").mkdir(parents=True)
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
|
||||
def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def]
|
||||
(workspace / "notes.txt").write_text("x\n")
|
||||
return "sideways"
|
||||
|
||||
proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
|
||||
assert proposer.propose(_facts(tmp_path, "injection")) is None
|
||||
assert "notes.txt" in proposer.reason
|
||||
|
||||
|
||||
def test_the_meta_agent_proposer_says_so_when_it_changed_nothing(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.proposer import HarnessXProposer
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
(configs / "injection").mkdir(parents=True)
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
proposer = HarnessXProposer(configs_dir=configs, model="m", agent=lambda **_kwargs: "nothing to do")
|
||||
assert proposer.propose(_facts(tmp_path, "injection")) is None
|
||||
assert proposer.reason == "no_change_proposed"
|
||||
|
||||
|
||||
def _facts(tmp_path: Path, family: str): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.proposer import FailureFacts
|
||||
|
||||
del tmp_path
|
||||
return FailureFacts(family=family, config=_config(family))
|
||||
@@ -1,209 +0,0 @@
|
||||
"""learning-harness task 3.8: publishing every number from its committed artifacts (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROADMAP = "# Roadmap\n\n## Overview\n\nSomething about the project.\n\n## Constraints\n\nCore dependencies stay empty.\n"
|
||||
|
||||
|
||||
def _baseline(out: Path, model: str, *, recall: float = 1.0, cost: float = 0.5) -> None:
|
||||
directory = out / "baseline" / model
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"k_runs": 5,
|
||||
"taxonomy_version": "1",
|
||||
"slices": ["vibe-py"],
|
||||
"cost_usd": cost,
|
||||
"floors": {"injection": {"family": "injection", "budget_flips": 1, "negative_flip_rate": 0.1, "k_runs": 5, "gates": True}},
|
||||
"metrics": {
|
||||
"injection": {
|
||||
"family": "injection",
|
||||
"taxonomy_version": "1",
|
||||
"scorable": 10,
|
||||
"unscorable": {"identical_twin": 2},
|
||||
"outcomes": {"pair_correct": round(recall * 10)}, # keep the fixture internally consistent
|
||||
"recall": recall,
|
||||
"silence": 1.0,
|
||||
"youden": recall,
|
||||
"directional_bias": 0.0,
|
||||
"leak_rate": 0.0,
|
||||
"fixed_fpr_recall": recall,
|
||||
"hierarchical_credit": False,
|
||||
}
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def test_publishing_twice_without_new_artifacts_is_byte_identical(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
measurements = tmp_path / "measurements"
|
||||
first = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
second = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
assert roadmap.read_text() == text
|
||||
assert first.table == second.table and first.families == second.families
|
||||
|
||||
|
||||
def test_every_number_carries_its_denominator_floor_and_cost(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9, cost=1.8)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
table = report.table
|
||||
assert "| injection |" in table
|
||||
assert "10" in table and "identical_twin: 2" in table # scorable and why the rest are not
|
||||
assert "taxonomy 1" in table and "budget 1" in table
|
||||
assert "0.20" in table # cost per pair-correct: 1.8 over 9
|
||||
assert "hierarchical partial credit could not apply" in table
|
||||
assert report.command and report.command in roadmap.read_text()
|
||||
|
||||
|
||||
def test_a_second_model_appears_beside_the_first(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9)
|
||||
_baseline(out, "deepseek-v4-pro", recall=1.0)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert "deepseek-v4-flash" in report.table and "deepseek-v4-pro" in report.table
|
||||
assert set(report.families) == {"injection"}
|
||||
|
||||
|
||||
def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
measurements = tmp_path / "m"
|
||||
measurements.mkdir()
|
||||
(measurements / "2026-09-06-mechanism-lever-split.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"slice": "vibe-py",
|
||||
"holdout_pairs": 17,
|
||||
"before": {"detected": 7, "youden": 0.0588},
|
||||
"after": {"detected": 5, "youden": -0.0588},
|
||||
}
|
||||
)
|
||||
)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
stale = "human holdout profile pair_correct 0 -> 6 of 17, Youden 0 -> +0.353, round 2 `no_proposals`."
|
||||
roadmap.write_text(f"{ROADMAP}\nMeasured 2026-09-05 with the vibe-py candidates: {stale}\n")
|
||||
report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
assert "-0.0588" in report.table or "-0.059" in report.table
|
||||
assert "5 of 17" in report.table or "detected 5" in report.table
|
||||
assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower()
|
||||
assert text.count("learning-harness:begin") == 1
|
||||
# Req 5.3 says *in place of* the earlier one. Leaving the old claim above the new section is how a document
|
||||
# ends up asserting two different numbers for the same measurement.
|
||||
assert stale not in text
|
||||
assert "superseded" in text and "learning-harness" in text
|
||||
assert "Measured 2026-09-05 with the vibe-py candidates:" in text, "the sentence is corrected, not deleted"
|
||||
before = roadmap.read_text()
|
||||
publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
assert roadmap.read_text() == before, "a second publish must be a no-op once the claim is corrected"
|
||||
|
||||
|
||||
def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.5)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9)
|
||||
publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
assert text.count("learning-harness:begin") == 1 and text.count("learning-harness:end") == 1
|
||||
assert "## Overview" in text and "## Constraints" in text # nothing else in the roadmap is disturbed
|
||||
assert "0.5" not in text.split("learning-harness:begin")[1] # the stale number is gone
|
||||
|
||||
|
||||
def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
measurements = tmp_path / "m"
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
written = sorted(path.name for path in measurements.iterdir())
|
||||
assert any(name.endswith("-learning-families.json") for name in written)
|
||||
payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text())
|
||||
# keyed by what identifies the run, so two slices of one model cannot collapse into one entry
|
||||
assert payload["models"]["deepseek-v4-flash (vibe-py)"]["injection"]["scorable"] == 10
|
||||
assert payload["command"]
|
||||
|
||||
|
||||
def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert report.families == () and "no baseline" in report.table.lower()
|
||||
assert "learning-harness:begin" in roadmap.read_text()
|
||||
|
||||
|
||||
def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_path: Path) -> None:
|
||||
"""Keyed by model alone, the vibe-py run silently replaced the agent-vfc run and the published table showed
|
||||
hand-written teaching code as if it were the corpus — the exact failure Req 3 exists to prevent."""
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
for key, slice_name, recall in (("deepseek-v4-flash-vibe-py", "vibe-py", 0.833), ("deepseek-v4-flash-agent-vfc", "agent-vfc", 0.444)):
|
||||
directory = out / "baseline" / key
|
||||
directory.mkdir(parents=True)
|
||||
(directory / "report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"slices": [slice_name],
|
||||
"k_runs": 5,
|
||||
"cost_usd": 1.0,
|
||||
"taxonomy_version": "1",
|
||||
"metrics": {
|
||||
"injection": {
|
||||
"family": "injection",
|
||||
"scorable": 10,
|
||||
"unscorable": {},
|
||||
"recall": recall,
|
||||
"silence": 1.0,
|
||||
"youden": recall,
|
||||
"hierarchical_credit": False,
|
||||
"outcomes": {"pair_correct": 5},
|
||||
}
|
||||
},
|
||||
"floors": {},
|
||||
}
|
||||
)
|
||||
)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text("# Roadmap\n")
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert "vibe-py" in report.table and "agent-vfc" in report.table
|
||||
assert report.table.count("| injection |") == 2, "one row per slice, not one row that survived"
|
||||
assert "0.833" in report.table and "0.444" in report.table
|
||||
@@ -1,94 +0,0 @@
|
||||
"""A refusal is data, not a `continue` (learning-harness review round 3, Req 5.1, 5.2).
|
||||
|
||||
`Refusal.degradation()` existed and nothing ever called it, so every path that declined to learn from a pair
|
||||
declined silently: the mechanism lever skipped a holdout-taught shape without a word, leave-one-out dropped
|
||||
non-teachers with no record, and the seeder gave a train-split identical twin the reason `split:train`, which
|
||||
is not true. A leak the run does not mention is a leak nobody reads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, split: str = "train", unscorable: str | None = None, tier: str = "seeded") -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
split=split,
|
||||
review_tier=tier,
|
||||
unscorable=unscorable,
|
||||
)
|
||||
|
||||
|
||||
def test_a_train_split_twin_is_refused_for_being_a_twin_not_for_its_split(tmp_path: Path) -> None:
|
||||
"""`split:train` on a train-split row is a lie, and it hides a corpus fact behind a split fact.
|
||||
|
||||
Two vibe-py rows are exactly this: `python-insecure-app-main-try-hack-me` and
|
||||
`vulnpy-deserialization-do-pickle-load` are train-split identical twins."""
|
||||
from openultrasast.semantic.seed import _skip_reason
|
||||
|
||||
assert _skip_reason(_case(tmp_path, "twin", split="train", unscorable="identical_twin")) == "identical_twin"
|
||||
assert _skip_reason(_case(tmp_path, "held", split="holdout")) == "split:holdout"
|
||||
assert _skip_reason(_case(tmp_path, "good", split="train")) is None
|
||||
|
||||
|
||||
def test_leave_one_out_records_the_pairs_it_would_not_learn_from(tmp_path: Path) -> None:
|
||||
"""Req 5.2: the refusal is recorded with the pair names, on a channel a caller can read."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
result = evaluate_loo([_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")])
|
||||
refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
|
||||
assert refusals, "leave-one-out excluded a holdout pair from teaching and said nothing"
|
||||
assert refusals[0]["pairs"] == ["hold-a"]
|
||||
assert result.teaching_pairs <= 1
|
||||
|
||||
|
||||
def test_the_mechanism_lever_records_every_shape_a_holdout_pair_taught(tmp_path: Path) -> None:
|
||||
"""The lever used to `continue` past a refused candidate; the run then showed a shorter list with no reason."""
|
||||
from openultrasast.improve.evolve import propose_mechanism_edits
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
cases = [_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")]
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
append_from_pair(
|
||||
candidates,
|
||||
Shape(
|
||||
language="python",
|
||||
sink_name="system",
|
||||
arity=1,
|
||||
source_positions=(0,),
|
||||
source_kinds=("request",),
|
||||
guard="none",
|
||||
mechanism="source_reaches_sink",
|
||||
),
|
||||
summary="s",
|
||||
cwe="CWE-78",
|
||||
pair="hold-a",
|
||||
provenance="human",
|
||||
tier="seeded",
|
||||
)
|
||||
refusals: list[dict[str, object]] = []
|
||||
edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl", degradations=refusals)
|
||||
assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == []
|
||||
assert [item["reason"] for item in refusals] == ["holdout_pair_refused"]
|
||||
assert refusals[0]["pairs"] == ["hold-a"]
|
||||
json.dumps(refusals) # a degradation has to survive the artifact it is written into
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5).
|
||||
|
||||
The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw
|
||||
deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and
|
||||
a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up.
|
||||
|
||||
Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision,
|
||||
the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether
|
||||
any of those decisions could be told from a coin flip.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openultrasast.learning.acceptance import reliability
|
||||
|
||||
|
||||
def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None:
|
||||
result = reliability(better=8, worse=0)
|
||||
assert result.better == 8 and result.worse == 0
|
||||
assert result.p_value < 0.05 and result.reliable is True
|
||||
|
||||
|
||||
def test_two_pairs_moving_the_right_way_is_not() -> None:
|
||||
"""n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing."""
|
||||
result = reliability(better=2, worse=0)
|
||||
assert result.p_value == 0.5 and result.reliable is False
|
||||
|
||||
|
||||
def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None:
|
||||
result = reliability(better=0, worse=0)
|
||||
assert result.p_value == 1.0 and result.reliable is False
|
||||
|
||||
|
||||
def test_an_even_split_is_never_reliable_however_large() -> None:
|
||||
assert reliability(better=20, worse=20).reliable is False
|
||||
|
||||
|
||||
def test_the_round_journals_the_reliability_of_its_own_decision() -> None:
|
||||
from openultrasast.learning.journal import RoundRecord
|
||||
|
||||
record = RoundRecord(
|
||||
round=1,
|
||||
family="injection",
|
||||
hypothesis="h",
|
||||
levers=("checklist",),
|
||||
predicted_affected=(),
|
||||
predicted_at_risk=(),
|
||||
outcome="accepted",
|
||||
train_better=8,
|
||||
train_worse=0,
|
||||
holdout_better=1,
|
||||
holdout_worse=0,
|
||||
)
|
||||
payload = record.to_dict()
|
||||
assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05
|
||||
assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so"
|
||||
@@ -1,247 +0,0 @@
|
||||
"""Three defects the round-4 review found, one test each (learning-harness, Req 3.2, 4.1, 9.1).
|
||||
|
||||
Two of them were the same shape as defects already rejected once: a proposer that cannot propose because the call
|
||||
is wrong and the error is swallowed, and a K-run floor that turned a documented command into a traceback. The
|
||||
third is the fixed-side half of the range defect — 28 rows resolve a span on the vulnerable side and none on the
|
||||
fixed side, so every finding there reads as outside the labeled function and the pair scores as silence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.harness_ext import has_harnessx
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
|
||||
from openultrasast.semantic.extra import has_semantic_extra
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED_OTHER = "from flask import request\n\n\ndef unrelated():\n return 'ok'\n"
|
||||
|
||||
|
||||
def _finding(line: int) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"tool-hunter:app.py:{line}",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["tool-hunter", "family:injection"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
# --- the meta-agent call -----------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable")
|
||||
def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`MetaAgent(inner_model=...)` wants a `ModelConfig`; a string reaches `self.inner_model.agentic(...)` and dies.
|
||||
|
||||
Every other HarnessX composition site in this repository builds the config properly. Here the mistake was
|
||||
invisible because a `# type: ignore` hid it from mypy and a bare `except Exception` turned it into the
|
||||
degradation reason `meta_agent_failed`, which reads exactly like the model declining."""
|
||||
from openultrasast.learning import proposer as proposer_module
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class _Agent:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
seen.update(kwargs)
|
||||
|
||||
async def evolve(self, **kwargs: object) -> Path:
|
||||
seen["evolve"] = kwargs
|
||||
workspace = kwargs["current_config"]
|
||||
assert isinstance(workspace, Path)
|
||||
(workspace / "checklist.md").write_text("- from the meta agent\n")
|
||||
return workspace
|
||||
|
||||
monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent)
|
||||
monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})")
|
||||
(tmp_path / "injection").mkdir()
|
||||
(tmp_path / "injection" / "checklist.md").write_text("- old\n")
|
||||
proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts())
|
||||
assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"}
|
||||
assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config"
|
||||
assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable")
|
||||
def test_a_meta_agent_built_wrongly_is_not_reported_as_a_declined_proposal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
""" "We called the agent wrong" and "the agent had nothing to propose" are different facts."""
|
||||
from openultrasast.learning import proposer as proposer_module
|
||||
|
||||
def _boom() -> object:
|
||||
raise TypeError("inner_model must be a ModelConfig")
|
||||
|
||||
monkeypatch.setattr(proposer_module, "_meta_agent_class", _boom)
|
||||
(tmp_path / "injection").mkdir()
|
||||
proposer = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m")
|
||||
with pytest.raises(TypeError, match="ModelConfig"):
|
||||
proposer.propose(_facts())
|
||||
|
||||
|
||||
def _facts(): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.detectors import FamilyConfig
|
||||
from openultrasast.learning.proposer import FailureFacts
|
||||
|
||||
return FailureFacts(
|
||||
family="injection",
|
||||
config=FamilyConfig(
|
||||
family="injection",
|
||||
version="0",
|
||||
prompt="p",
|
||||
checklist="- old",
|
||||
tools=("read_file",),
|
||||
max_steps=1,
|
||||
max_cost_usd=0.1,
|
||||
max_chars=100,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# --- the pairs command in its documented form --------------------------------
|
||||
|
||||
|
||||
def test_the_pairs_hunter_path_runs_without_a_k_runs_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The floor is a default, not a trap: `ousast pairs --hunter` must work as documented."""
|
||||
from openultrasast.cli import main
|
||||
|
||||
(tmp_path / "a-v.py").write_text(VULN)
|
||||
(tmp_path / "a-f.py").write_text(FIXED_OTHER)
|
||||
(tmp_path / "pairs.toml").write_text(
|
||||
'[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n'
|
||||
'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n'
|
||||
)
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
|
||||
assert main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted"]) == 0
|
||||
|
||||
|
||||
def test_a_k_below_the_floor_names_the_flag_instead_of_raising(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.cli import main
|
||||
|
||||
(tmp_path / "pairs.toml").write_text("")
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
|
||||
with pytest.raises(SystemExit, match="k-runs"):
|
||||
main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "1"])
|
||||
|
||||
|
||||
# --- the fixed side of a trap pair -------------------------------------------
|
||||
|
||||
|
||||
def test_a_trap_pair_is_scored_against_the_function_its_fixed_side_really_holds(tmp_path: Path) -> None:
|
||||
"""A Real-Vuln-Benchmark trap takes its fixed side from a different, correctly guarded handler.
|
||||
|
||||
With no span for it, every fixed-side finding counts as outside the labeled function and the pair scores as
|
||||
silence — the same flattering direction as the range defect, for the 28 rows that resolve one side only."""
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
(tmp_path / "v.py").write_text(VULN)
|
||||
(tmp_path / "f.py").write_text(FIXED_OTHER)
|
||||
case = PairCase(
|
||||
name="trap",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection"
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
fix_function="unrelated",
|
||||
)
|
||||
score = score_pair_family(
|
||||
case,
|
||||
"injection",
|
||||
[[_finding(6)]],
|
||||
[[_finding(5)]],
|
||||
ranges={"app.py": (("run", 5, 6),)},
|
||||
fixed_ranges={"app.py": (("unrelated", 4, 5),)},
|
||||
taxonomy=load_families(),
|
||||
)
|
||||
assert score.outcome == "both_flagged", "the finding is inside the fixed side's own handler; that is a leak"
|
||||
|
||||
|
||||
def test_a_row_whose_fixed_side_resolves_no_span_says_so_instead_of_scoring_silent(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
(tmp_path / "v.py").write_text(VULN)
|
||||
(tmp_path / "f.py").write_text(FIXED_OTHER)
|
||||
case = PairCase(
|
||||
name="orphan",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection"
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
)
|
||||
score = score_pair_family(
|
||||
case,
|
||||
"injection",
|
||||
[[_finding(6)]],
|
||||
[[]],
|
||||
ranges={"app.py": (("run", 5, 6),)},
|
||||
fixed_ranges={"app.py": (("unrelated", 4, 5),)},
|
||||
taxonomy=load_families(),
|
||||
)
|
||||
assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label:fixed"
|
||||
|
||||
|
||||
def test_the_trap_rows_carry_their_fixed_side_function_in_the_catalog() -> None:
|
||||
"""`fix_function` lived only in the recipes; the scorer could not see which handler the fixed side holds."""
|
||||
cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))}
|
||||
assert cases["threatbyte-api-v1-get"].fix_function == "post"
|
||||
assert cases["threatbyte-api-v1-delete"].fix_function == "post"
|
||||
assert cases["vampi-users-update-password"].fix_function == "register_user"
|
||||
assert cases["vampi-books-get-by-title"].fix_function == "", "a row whose two sides are one function names none"
|
||||
|
||||
|
||||
@pytest.mark.semantic
|
||||
@pytest.mark.skipif(not has_semantic_extra(), reason="resolving a span on both sides needs the tree-sitter grammars")
|
||||
def test_no_vendored_row_resolves_one_side_and_not_the_other() -> None:
|
||||
"""The corpus-wide form of the same defect: 28 rows resolved a vulnerable span and no fixed-side span.
|
||||
|
||||
Every one of them therefore scored the fixed side as silent by construction, whatever the detector said. The
|
||||
catalog generator now derives `fix_function` from the fixed excerpt itself, so the span exists and the leak is
|
||||
visible."""
|
||||
import tempfile
|
||||
|
||||
from openultrasast.learning.scoring import _spans
|
||||
from openultrasast.pairs import _materialize_side, _overlay_scan
|
||||
|
||||
orphans = []
|
||||
for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)):
|
||||
if case.unscorable:
|
||||
continue
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
vuln = _materialize_side(Path(scratch) / "vuln", case, side="vuln")
|
||||
fixed = _materialize_side(Path(scratch) / "fixed", case, side="fixed")
|
||||
vuln_ranges, fixed_ranges = _overlay_scan(vuln).ranges, _overlay_scan(fixed).ranges
|
||||
if _spans(case, vuln_ranges) and not _spans(case, fixed_ranges, function=case.fix_function):
|
||||
orphans.append(case.name)
|
||||
assert orphans == [], f"rows whose fixed side can only ever score as silence: {orphans}"
|
||||
@@ -1,428 +0,0 @@
|
||||
"""What a round must prove before it is kept (learning-harness review round 3, Req 4.1, 5.2, 9.5, 9.6, 9.8).
|
||||
|
||||
The round loop was accepting and rejecting on numbers that did not mean what their names said: the
|
||||
cross-family sweep counted every failing pair rather than the pairs this change broke, unscorable rows
|
||||
voted against a round they cannot judge, cost was metered once after everything had been paid for, and a
|
||||
stage that raised left no record at all. Each test here fails on exactly one of those.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.detectors import FamilyConfig, write_default_configs
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.journal import Archive, LearningJournal
|
||||
from openultrasast.learning.proposer import Proposal, ScriptedProposer
|
||||
from openultrasast.learning.rounds import NoiseFloor, run_learning_round
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection", unscorable: str | None = None) -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
split=split,
|
||||
review_tier="seeded",
|
||||
unscorable=unscorable,
|
||||
)
|
||||
|
||||
|
||||
def _finding(family: str) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="detector:app.py:6",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=6,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", "detector:x@0"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _proposal(text: str = "- ask about quoting\n") -> Proposal:
|
||||
return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("train-a",))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def world(tmp_path: Path): # type: ignore[no-untyped-def]
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
return {
|
||||
"configs": configs,
|
||||
"cases": [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")],
|
||||
"journal": LearningJournal(tmp_path / "journal.jsonl"),
|
||||
"archive": Archive(tmp_path / "archive.jsonl"),
|
||||
"out": tmp_path / "out",
|
||||
"floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)},
|
||||
"root": tmp_path,
|
||||
}
|
||||
|
||||
|
||||
def _run(world, proposal, build, **kwargs): # type: ignore[no-untyped-def]
|
||||
return run_learning_round(
|
||||
world["cases"],
|
||||
family="injection",
|
||||
taxonomy=load_families(),
|
||||
configs_dir=world["configs"],
|
||||
proposer=ScriptedProposer([proposal] if proposal else []),
|
||||
scan_factory=build,
|
||||
model="m",
|
||||
journal=world["journal"],
|
||||
archive=world["archive"],
|
||||
floors=world["floors"],
|
||||
out_dir=world["out"],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _injection_improves(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
"""Injection detects only once the proposed checklist lands; every other family finds nothing, ever."""
|
||||
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln" or config.family != "injection":
|
||||
return []
|
||||
return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else []
|
||||
|
||||
return scan
|
||||
|
||||
|
||||
# --- the sweep ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_family_that_was_already_failing_does_not_reject_the_round(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Req 9.5 counts *negative flips*, not absolute failures.
|
||||
|
||||
On the real corpus most pairs fail at round zero, so counting failures rejects every round for damage it
|
||||
did not do — and the harness then reports that it cannot improve."""
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
|
||||
assert record.sweep["access_control"] == 0, "access control failed before and after; this change flipped nothing"
|
||||
assert record.outcome == "accepted" and record.reason == "improved"
|
||||
|
||||
|
||||
def test_a_family_this_change_really_broke_still_rejects_the_round(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""The other arm: a pair that was correct before and is not after is a flip, and one is over the budget."""
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
landed = {"yes": False}
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln":
|
||||
return []
|
||||
if config.family == "access_control":
|
||||
return [] if landed["yes"] else [_finding("access_control")]
|
||||
if config.checklist.strip() == "- ask about quoting":
|
||||
landed["yes"] = True
|
||||
return [_finding("injection")]
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
record = _run(world, _proposal(), build, sweep_families=("access_control",))
|
||||
assert record.sweep["access_control"] == 1
|
||||
assert record.outcome == "rejected" and record.reason == "collateral_regression"
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
|
||||
|
||||
def test_an_unscorable_row_never_votes_in_the_sweep(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Req 4.1: an unscorable row leaves every denominator, and it certainly does not reject a round."""
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control", unscorable="identical_twin"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=0, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
|
||||
assert record.sweep["access_control"] == 0 and record.outcome == "accepted"
|
||||
|
||||
|
||||
def test_attribution_names_the_pairs_this_change_flipped_and_no_others(world) -> None: # type: ignore[no-untyped-def]
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
_run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
|
||||
attribution = json.loads((world["out"] / "rounds" / "1" / "attribution.json").read_text())
|
||||
assert attribution["flipped_predicted"] == ["hold-a", "train-a"]
|
||||
assert attribution["flipped_unpredicted"] == [], "a pair that was already failing was never flipped by this round"
|
||||
|
||||
|
||||
# --- cost, exceptions, the archive ------------------------------------------
|
||||
|
||||
|
||||
def test_the_cap_stops_a_round_before_it_pays_for_the_next_stage(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Req 9.8 says stop, revert and record. Metering once, after every stage, can only ever record."""
|
||||
stages: list[str] = []
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
stages.append(f"{config.family}:{root.name}")
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
spend = {"usd": 0.0}
|
||||
|
||||
def spent() -> float:
|
||||
spend["usd"] += 1.0
|
||||
return spend["usd"]
|
||||
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
|
||||
record = _run(world, _proposal(), build, cost_cap_usd=0.5, spent_usd=spent, sweep_families=("access_control",))
|
||||
assert record.outcome == "reverted_cost" and record.reason == "cost_cap_exceeded"
|
||||
assert "access_control:vuln" not in stages, "the sweep ran after the cap was already crossed"
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
|
||||
|
||||
def test_a_stage_that_raises_is_reverted_journalled_and_re_raised(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""An endpoint failure mid-round is an outcome. Losing the round record loses the money it already spent."""
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if config.checklist.strip() == "- ask about quoting":
|
||||
raise RuntimeError("endpoint said no")
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
with pytest.raises(RuntimeError, match="endpoint said no"):
|
||||
_run(world, _proposal(), build)
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()]
|
||||
assert outcomes == [("reverted", "stage_failed: RuntimeError")] # the exception type is part of the record
|
||||
assert (world["out"] / "rounds" / "1" / "scores.json").is_file()
|
||||
|
||||
|
||||
def test_a_rejected_version_is_archived_so_a_pair_it_alone_wins_is_not_lost(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Req 9.7: per-pair winners, or the loop collapses onto one lineage and forgets what a rejected try could do.
|
||||
|
||||
This round wins both of its own pairs and is still rejected, because it broke another family. Version 1 is the
|
||||
only configuration that ever got those two pairs right; dropping it because the round as a whole lost is how a
|
||||
search collapses onto one lineage."""
|
||||
world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
landed = {"yes": False}
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln":
|
||||
return []
|
||||
if config.family == "access_control":
|
||||
return [] if landed["yes"] else [_finding("access_control")]
|
||||
if config.checklist.strip() == "- ask about quoting":
|
||||
landed["yes"] = True
|
||||
return [_finding("injection")]
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
record = _run(world, _proposal(), build, sweep_families=("access_control",))
|
||||
assert record.outcome == "rejected" and record.reason == "collateral_regression"
|
||||
assert world["archive"].winners("injection") == {"train-a": "1", "hold-a": "1"}
|
||||
assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text(), "a rejected round is not versioned"
|
||||
|
||||
|
||||
# --- the holdout refusal -----------------------------------------------------
|
||||
|
||||
|
||||
def test_a_proposal_that_names_a_holdout_pair_is_refused_by_name(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Req 5.2: a candidate that touches a holdout pair is refused and the refusal names the pairs."""
|
||||
record = _run(
|
||||
world,
|
||||
Proposal(
|
||||
hypothesis="memorize the answer",
|
||||
lever="memory",
|
||||
change={"hard_negatives.jsonl": json.dumps({"pair": "hold-a", "note": "always flag"}) + "\n"},
|
||||
predicted_affected=("hold-a",),
|
||||
),
|
||||
_injection_improves,
|
||||
)
|
||||
assert record.outcome == "refused_holdout"
|
||||
assert "hold-a" in record.reason
|
||||
assert (world["configs"] / "injection" / "hard_negatives.jsonl").read_text() == ""
|
||||
|
||||
|
||||
def test_a_proposal_that_only_names_train_pairs_is_not_refused(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(
|
||||
world,
|
||||
Proposal(
|
||||
hypothesis="remember this shape",
|
||||
lever="memory",
|
||||
change={"hard_negatives.jsonl": json.dumps({"pair": "train-a", "note": "shape"}) + "\n"},
|
||||
predicted_affected=("train-a",),
|
||||
),
|
||||
_injection_improves,
|
||||
)
|
||||
assert record.outcome != "refused_holdout"
|
||||
|
||||
|
||||
# --- round zero --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_round_zero_measures_each_family_with_its_own_detector(tmp_path: Path) -> None:
|
||||
"""Req 8.2. One detector for every family means the generalist scores them all, and abstention earns no credit
|
||||
(`related(X, "unknown")` is lateral), so every family but `unknown` baselines at a structural zero."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
seen: list[str] = []
|
||||
|
||||
def factory(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
seen.append(config.family)
|
||||
return [_finding(config.family)] if root.name == "vuln" else []
|
||||
|
||||
return scan
|
||||
|
||||
cases = [_case(tmp_path, "inj", split="train"), _case(tmp_path, "acl", split="train", family="access_control")]
|
||||
report = run_baseline(
|
||||
cases,
|
||||
taxonomy=load_families(),
|
||||
configs_dir=configs,
|
||||
scan_factory=factory,
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
k_runs=3,
|
||||
)
|
||||
assert set(seen) == {"injection", "access_control"}, "each family must be measured with its own configuration"
|
||||
assert report.metrics["injection"]["recall"] == 1.0 and report.metrics["access_control"]["recall"] == 1.0
|
||||
|
||||
|
||||
def test_round_zero_reports_each_slice_separately(tmp_path: Path) -> None:
|
||||
"""Req 8.4: vibe-py and agent-vfc are different worlds; one pooled number hides which one moved."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
web = _case(tmp_path, "web", split="train")
|
||||
agent = _case(tmp_path, "agent", split="train")
|
||||
object.__setattr__(agent, "slice", "agent-vfc")
|
||||
|
||||
def factory(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
return [_finding(config.family)] if root.name == "vuln" else []
|
||||
|
||||
return scan
|
||||
|
||||
report = run_baseline(
|
||||
[web, agent],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=configs,
|
||||
scan_factory=factory,
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
k_runs=3,
|
||||
)
|
||||
assert set(report.per_slice) == {"vibe-py/injection", "agent-vfc/injection"}
|
||||
payload = json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())
|
||||
assert set(payload["per_slice"]) == {"vibe-py/injection", "agent-vfc/injection"}
|
||||
|
||||
|
||||
def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None:
|
||||
"""The floor exists because a single run misses most real changes; a baseline measured at K=1 is a noise floor
|
||||
of zero and every later round then inherits a budget of nothing."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
with pytest.raises(ValueError, match="three runs"):
|
||||
run_baseline(
|
||||
[_case(tmp_path, "inj", split="train")],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=configs,
|
||||
scan_factory=lambda config: lambda root: [],
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
k_runs=1,
|
||||
)
|
||||
|
||||
|
||||
def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start.
|
||||
|
||||
Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a
|
||||
provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not
|
||||
look identical to a model that considered the facts and declined."""
|
||||
|
||||
class Mute:
|
||||
reason = "meta_agent_failed: RuntimeError"
|
||||
|
||||
def propose(self, facts): # type: ignore[no-untyped-def]
|
||||
del facts
|
||||
return None
|
||||
|
||||
from openultrasast.learning.rounds import run_learning_round
|
||||
|
||||
record = run_learning_round(
|
||||
world["cases"],
|
||||
family="injection",
|
||||
taxonomy=load_families(),
|
||||
configs_dir=world["configs"],
|
||||
proposer=Mute(),
|
||||
scan_factory=_injection_improves,
|
||||
model="m",
|
||||
journal=world["journal"],
|
||||
archive=world["archive"],
|
||||
floors=world["floors"],
|
||||
out_dir=world["out"],
|
||||
)
|
||||
assert record.outcome == "rejected"
|
||||
assert record.reason == "no_proposal: meta_agent_failed: RuntimeError"
|
||||
|
||||
|
||||
def test_a_round_that_dies_before_it_applies_anything_is_still_journalled(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Measured: round 5 died on a network read during its *first* stage and left no record at all.
|
||||
|
||||
The exception path was wrapped around the stages after the proposal is applied, because those are the ones that
|
||||
have something to revert. The ones before it had already spent money."""
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
raise ConnectionError("the body stopped arriving")
|
||||
|
||||
return scan
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
_run(world, _proposal(), build)
|
||||
outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()]
|
||||
assert outcomes == [("reverted", "stage_failed: ConnectionError")]
|
||||
assert (world["out"] / "rounds" / "1").is_dir()
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
@@ -1,230 +0,0 @@
|
||||
"""learning-harness task 3.7: one evolve round, end to end (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.detectors import FamilyConfig, write_default_configs
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.journal import Archive, LearningJournal
|
||||
from openultrasast.learning.proposer import Proposal, ScriptedProposer
|
||||
from openultrasast.learning.rounds import NoiseFloor
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection") -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
split=split,
|
||||
review_tier="seeded",
|
||||
)
|
||||
|
||||
|
||||
def _finding(family: str) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="detector:app.py:6",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=6,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", "detector:x@0"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _factory(finds: dict[str, bool]): # type: ignore[no-untyped-def]
|
||||
"""A detector whose behaviour depends on the configuration version it was built from."""
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln":
|
||||
return []
|
||||
return [_finding(config.family)] if finds.get(config.checklist.strip(), False) else []
|
||||
|
||||
return scan
|
||||
|
||||
return build
|
||||
|
||||
|
||||
def _proposal(text: str) -> Proposal:
|
||||
return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("injection",))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def world(tmp_path: Path): # type: ignore[no-untyped-def]
|
||||
configs = tmp_path / "configs"
|
||||
write_default_configs(configs, load_families(), prompt="p", version="0")
|
||||
(configs / "injection" / "checklist.md").write_text("- old\n")
|
||||
cases = [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")]
|
||||
return {
|
||||
"configs": configs,
|
||||
"cases": cases,
|
||||
"journal": LearningJournal(tmp_path / "journal.jsonl"),
|
||||
"archive": Archive(tmp_path / "archive.jsonl"),
|
||||
"out": tmp_path / "out",
|
||||
"floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)},
|
||||
}
|
||||
|
||||
|
||||
def _run(world, proposal: Proposal | None, finds: dict[str, bool], **kwargs): # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.rounds import run_learning_round
|
||||
|
||||
return run_learning_round(
|
||||
world["cases"],
|
||||
family="injection",
|
||||
taxonomy=load_families(),
|
||||
configs_dir=world["configs"],
|
||||
proposer=ScriptedProposer([proposal] if proposal else []),
|
||||
scan_factory=_factory(finds),
|
||||
model="m",
|
||||
journal=world["journal"],
|
||||
archive=world["archive"],
|
||||
floors=world["floors"],
|
||||
out_dir=world["out"],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_a_change_that_helps_is_accepted_versioned_and_journalled(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True})
|
||||
assert record.outcome == "accepted" and record.family == "injection"
|
||||
assert record.target_train_delta > 0 and record.target_holdout_delta > 0
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- ask about quoting\n"
|
||||
assert 'version = "1"' in (world["configs"] / "injection" / "family.toml").read_text()
|
||||
assert [item.outcome for item in world["journal"].rounds()] == ["accepted"]
|
||||
assert world["archive"].winners("injection")
|
||||
|
||||
|
||||
def test_a_change_that_helps_nothing_is_rejected_and_reverted(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, _proposal("- useless\n"), {})
|
||||
assert record.outcome == "rejected" and record.reason == "tie"
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text()
|
||||
assert world["journal"].rejected_buffer("injection") == ("ask about quoting",)
|
||||
|
||||
|
||||
def test_a_round_is_rejected_when_the_sweep_finds_another_family_below_its_floor(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""The sweep counts pairs this change turned from right to wrong in another family (Req 9.5).
|
||||
|
||||
It used to count every failing pair, which rejected a round for damage it did not do; the pair below is right
|
||||
before the change and wrong after it, which is the only thing that may reject a round on another family's
|
||||
behalf. The other arm — a family that was already failing must not reject — lives in
|
||||
`tests/test_learning_round_evidence.py`.
|
||||
"""
|
||||
world["cases"].append(_case(world["configs"].parent, "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
landed = {"yes": False}
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln":
|
||||
return []
|
||||
if config.family == "access_control":
|
||||
return [] if landed["yes"] else [_finding("access_control")]
|
||||
if config.checklist.strip() == "- ask about quoting":
|
||||
landed["yes"] = True # the injection change lands, and access control stops finding its pair
|
||||
return [_finding("injection")]
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
from openultrasast.learning.rounds import run_learning_round
|
||||
|
||||
record = run_learning_round(
|
||||
world["cases"],
|
||||
family="injection",
|
||||
taxonomy=load_families(),
|
||||
configs_dir=world["configs"],
|
||||
proposer=ScriptedProposer([_proposal("- ask about quoting\n")]),
|
||||
scan_factory=build,
|
||||
model="m",
|
||||
journal=world["journal"],
|
||||
archive=world["archive"],
|
||||
floors=world["floors"],
|
||||
out_dir=world["out"],
|
||||
sweep_families=("access_control",),
|
||||
)
|
||||
assert record.outcome == "rejected" and record.reason == "collateral_regression"
|
||||
assert record.sweep["access_control"] == 1
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
|
||||
|
||||
def test_a_round_over_its_cost_cap_is_reverted_and_says_so(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}, cost_cap_usd=0.0, spent_usd=lambda: 1.0)
|
||||
assert record.outcome == "reverted_cost" and record.cost_usd == 1.0
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
|
||||
|
||||
def test_a_proposal_that_reaches_beyond_its_family_never_runs(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, Proposal(hypothesis="h", lever="checklist", change={"../escape.md": "x"}), {})
|
||||
assert record.outcome == "rejected" and "outside" in record.reason # the holdout refusal has its own test
|
||||
assert not (world["configs"].parent / "escape.md").exists()
|
||||
|
||||
|
||||
def test_a_round_with_no_proposal_is_recorded_rather_than_skipped(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, None, {})
|
||||
assert record.outcome == "rejected" and record.reason == "no_proposal"
|
||||
assert [item.round for item in world["journal"].rounds()] == [1]
|
||||
|
||||
|
||||
def test_the_round_directory_holds_what_a_replay_needs(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(
|
||||
world,
|
||||
_proposal("- ask about quoting\n"),
|
||||
{"- ask about quoting": True},
|
||||
trajectories=lambda: [{"pair": "train-a", "prompt": "api_key = " + "'" + "B" * 24 + "'"}],
|
||||
)
|
||||
directory = world["out"] / "rounds" / "1"
|
||||
assert (directory / "proposal.json").is_file() and (directory / "scores.json").is_file()
|
||||
attribution = json.loads((directory / "attribution.json").read_text())
|
||||
assert attribution["flipped_predicted"] == ["hold-a", "train-a"] # the round is deterministic; one exact answer
|
||||
assert attribution["flipped_unpredicted"] == []
|
||||
trajectories = (directory / "trajectories.jsonl").read_text()
|
||||
assert "REDACTED" in trajectories and "B" * 24 not in trajectories
|
||||
assert record.attribution.precision == 1.0
|
||||
|
||||
|
||||
def test_a_round_number_is_never_reused(world) -> None: # type: ignore[no-untyped-def]
|
||||
from openultrasast.learning.journal import JournalError
|
||||
|
||||
_run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True})
|
||||
(world["out"] / "rounds" / "2").mkdir(parents=True)
|
||||
with pytest.raises(JournalError, match="2"):
|
||||
_run(world, _proposal("- another\n"), {})
|
||||
|
||||
|
||||
def test_every_stage_scores_at_least_three_runs_per_pair(world) -> None: # type: ignore[no-untyped-def]
|
||||
with pytest.raises(ValueError, match="three"):
|
||||
_run(world, _proposal("- x\n"), {}, k_runs=2)
|
||||
@@ -1,377 +0,0 @@
|
||||
"""learning-harness task 3.3: round zero and the per-family noise floor (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, family: str = "injection", slice_name: str = "vibe-py", unscorable: str | None = None) -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice=slice_name,
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
review_tier="seeded",
|
||||
)
|
||||
|
||||
|
||||
def _finding(family: str) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="detector:app.py:6",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=6,
|
||||
function_name="run",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", "detector:x@0"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _is_vulnerable_side(root: Path) -> bool:
|
||||
"""The materialized side, by directory name only.
|
||||
|
||||
An earlier version also matched "-v" anywhere in the path, which the random temporary directory name
|
||||
sometimes contained, so the detector "found" the bug on the fixed side too and the pair flipped. A
|
||||
flaky harness is indistinguishable from a flaky detector, which is precisely what a noise floor must
|
||||
not measure.
|
||||
"""
|
||||
return root.name == "vuln"
|
||||
|
||||
|
||||
def _steady(root: Path) -> list[StaticFinding]:
|
||||
return [_finding("injection")] if _is_vulnerable_side(root) else []
|
||||
|
||||
|
||||
def _always(scan): # type: ignore[no-untyped-def]
|
||||
"""Round zero asks for a detector per family; these tests deliberately give every family the same one so the
|
||||
noise-floor arithmetic is the only thing under test."""
|
||||
|
||||
def build(_config): # type: ignore[no-untyped-def]
|
||||
return scan
|
||||
|
||||
return build
|
||||
|
||||
|
||||
def _flaky(counter: list[int]): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if not _is_vulnerable_side(root):
|
||||
return []
|
||||
counter.append(1)
|
||||
return [_finding("injection")] if len(counter) % 2 else []
|
||||
|
||||
return scan
|
||||
|
||||
|
||||
def test_round_zero_writes_a_configuration_for_every_family(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.detectors import load_family_configs
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "a")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
)
|
||||
configs = load_family_configs(tmp_path / "configs", taxonomy)
|
||||
assert set(configs) == {family.id for family in taxonomy.families}
|
||||
assert all(config.version == "0" for config in configs.values())
|
||||
assert report.model == "m" and report.k_runs == 5
|
||||
|
||||
|
||||
def test_a_steady_detector_has_a_zero_noise_floor_and_a_flaky_one_does_not(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
steady = run_baseline(
|
||||
[_case(tmp_path, "a"), _case(tmp_path, "b")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "c1",
|
||||
scan_factory=_always(_steady),
|
||||
model="m",
|
||||
out_dir=tmp_path / "o1",
|
||||
)
|
||||
floor = steady.floors["injection"]
|
||||
assert floor.k_runs == 5 and floor.negative_flip_rate == 0.0 and floor.budget_flips == 0 and floor.gates is True
|
||||
flaky = run_baseline(
|
||||
[_case(tmp_path, "a"), _case(tmp_path, "b")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "c2",
|
||||
scan_factory=_always(_flaky([])),
|
||||
model="m",
|
||||
out_dir=tmp_path / "o2",
|
||||
)
|
||||
assert flaky.floors["injection"].negative_flip_rate > 0.0 and flaky.floors["injection"].budget_flips >= 1
|
||||
|
||||
|
||||
def test_the_memory_family_is_measured_but_never_gates(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "m", family="memory", slice_name="vfc")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
slices=("vfc",),
|
||||
)
|
||||
assert "memory" in report.floors and report.floors["memory"].gates is False
|
||||
assert report.metrics["memory"]["scorable"] == 1
|
||||
|
||||
|
||||
def test_the_default_selection_is_the_web_slices(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
cases = [_case(tmp_path, "web", slice_name="vibe-py"), _case(tmp_path, "c", family="memory", slice_name="vfc")]
|
||||
report = run_baseline(
|
||||
cases, taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(_steady), model="m", out_dir=tmp_path / "out"
|
||||
)
|
||||
assert set(report.metrics) == {"injection"} and "memory" not in report.floors
|
||||
|
||||
|
||||
def test_artifacts_are_keyed_by_model_so_another_model_can_be_compared(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import compare_baselines, run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
out = tmp_path / "out"
|
||||
for index, model in enumerate(("deepseek-v4-flash", "deepseek-v4-pro")):
|
||||
run_baseline(
|
||||
[_case(tmp_path, f"a{index}")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model=model,
|
||||
out_dir=out,
|
||||
)
|
||||
written = sorted(path.name for path in (out / "baseline").iterdir())
|
||||
assert written == ["deepseek-v4-flash", "deepseek-v4-pro"]
|
||||
floors = json.loads((out / "baseline" / "deepseek-v4-flash" / "noise-floors.json").read_text())
|
||||
assert floors["model"] == "deepseek-v4-flash" and floors["floors"]["injection"]["k_runs"] == 5
|
||||
report = json.loads((out / "baseline" / "deepseek-v4-flash" / "report.json").read_text())
|
||||
assert report["metrics"]["injection"]["recall"] == 1.0 and report["taxonomy_version"]
|
||||
table = compare_baselines(out)
|
||||
assert set(table) == {"deepseek-v4-flash", "deepseek-v4-pro"}
|
||||
assert table["deepseek-v4-pro"]["injection"]["youden"] == 1.0
|
||||
# a second baseline never rewrites a configuration
|
||||
assert (tmp_path / "configs" / "injection" / "family.toml").read_text().count("version") == 1
|
||||
|
||||
|
||||
def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
calls: list[Path] = []
|
||||
|
||||
def counting(root: Path) -> list[StaticFinding]:
|
||||
calls.append(root)
|
||||
return _steady(root)
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path, "twin")
|
||||
twin = PairCase(**{**case.__dict__, "unscorable": "identical_twin"})
|
||||
report = run_baseline(
|
||||
[twin], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(counting), model="m", out_dir=tmp_path / "out"
|
||||
)
|
||||
assert report.metrics["injection"]["scorable"] == 0
|
||||
assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_round_zero_records_what_it_spent(tmp_path: Path) -> None:
|
||||
"""Publishing puts a cost beside every number, so the baseline has to know what it cost."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "a")],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
spent_usd=lambda: 1.25,
|
||||
)
|
||||
assert report.cost_usd == 1.25
|
||||
assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25
|
||||
|
||||
|
||||
def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_run(tmp_path: Path) -> None:
|
||||
"""One transient error must not cost a run that has already paid for hundreds of calls.
|
||||
|
||||
Round zero is the most expensive thing this harness does. A provider hiccup two thirds of the way through used
|
||||
to raise straight out of `run_baseline`, and everything measured so far went with it — so the honest response
|
||||
to a flaky endpoint was to run the whole thing again and hope."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
seen: list[str] = []
|
||||
|
||||
def build(_config): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
seen.append(root.name)
|
||||
if len(seen) == 3:
|
||||
raise TimeoutError("the endpoint hung up")
|
||||
return _steady(root)
|
||||
|
||||
return scan
|
||||
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "a"), _case(tmp_path, "b")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=build,
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
k_runs=3,
|
||||
)
|
||||
assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured"
|
||||
assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1}
|
||||
assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file()
|
||||
|
||||
|
||||
def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
|
||||
"""Artifacts are keyed by what identifies the run, and the slice selection is part of that.
|
||||
|
||||
Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model
|
||||
alone, the second run silently replaces the first and the published table shows one of them as if it were
|
||||
everything."""
|
||||
from openultrasast.learning.rounds import compare_baselines, run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
out = tmp_path / "out"
|
||||
web = run_baseline(
|
||||
[_case(tmp_path, "w", slice_name="vibe-py")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="deepseek-v4-flash",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
)
|
||||
memory = run_baseline(
|
||||
[_case(tmp_path, "m", family="memory", slice_name="vfc")],
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="deepseek-v4-flash",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
slices=("vfc",),
|
||||
)
|
||||
assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"}
|
||||
written = sorted(path.name for path in (out / "baseline").iterdir())
|
||||
assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written
|
||||
table = compare_baselines(out)
|
||||
assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"}
|
||||
assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"]
|
||||
|
||||
|
||||
def test_round_zero_keeps_every_pair_it_has_already_paid_for(tmp_path: Path) -> None:
|
||||
"""A four-hour run that is interrupted must not lose four hours of paid calls.
|
||||
|
||||
Measured the hard way: the 176-pair vfc baseline hit its timeout and wrote nothing, because scores were
|
||||
accumulated in memory and persisted once at the end. An exception was already handled; a timeout, a kill and a
|
||||
machine going away were not, and they cost the same money."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
out = tmp_path / "out"
|
||||
scored: list[str] = []
|
||||
|
||||
def build(_config): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
scored.append(root.name)
|
||||
# K=3 means three vulnerable-side scans per pair, so this gives up part way through the third pair
|
||||
if len([item for item in scored if item == "vuln"]) > 6:
|
||||
raise KeyboardInterrupt("the operator gave up")
|
||||
return _steady(root)
|
||||
|
||||
return scan
|
||||
|
||||
cases = [_case(tmp_path, name) for name in ("a", "b", "c", "d")]
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
run_baseline(
|
||||
cases,
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=build,
|
||||
model="m",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
)
|
||||
checkpoint = out / "baseline" / "m" / "scores.jsonl"
|
||||
assert checkpoint.is_file(), "the pairs already measured were not written down"
|
||||
rows = [json.loads(line) for line in checkpoint.read_text().splitlines() if line.strip()]
|
||||
assert [row["pair"] for row in rows] == ["a", "b"]
|
||||
assert all(row["outcome"] == "pair_correct" for row in rows)
|
||||
|
||||
|
||||
def test_a_resumed_baseline_does_not_pay_twice(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
taxonomy = load_families()
|
||||
out = tmp_path / "out"
|
||||
cases = [_case(tmp_path, name) for name in ("a", "b")]
|
||||
run_baseline(
|
||||
cases,
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="m",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
)
|
||||
calls: list[str] = []
|
||||
|
||||
def counting(_config): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
calls.append(root.name)
|
||||
return _steady(root)
|
||||
|
||||
return scan
|
||||
|
||||
report = run_baseline(
|
||||
cases,
|
||||
taxonomy=taxonomy,
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=counting,
|
||||
model="m",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
resume=True,
|
||||
)
|
||||
assert calls == [], "every pair was already measured and paid for"
|
||||
assert report.metrics["injection"]["scorable"] == 2
|
||||
@@ -1,109 +0,0 @@
|
||||
"""learning-harness task 3.4: family detectors, verifiers and tags in a scan (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
APP = (
|
||||
"from flask import request\n\n\n"
|
||||
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
|
||||
" return Book.query.filter_by(book_title=title).first()\n\n\n"
|
||||
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
|
||||
" owner = request.args.get('owner')\n"
|
||||
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n"
|
||||
)
|
||||
|
||||
|
||||
def _repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, toml: str, *, configs: bool = True) -> Path:
|
||||
from openultrasast.learning.detectors import write_default_configs
|
||||
from openultrasast.learning.families import load_families
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(APP)
|
||||
(repo / "openultrasast.toml").write_text(toml)
|
||||
if configs: # round zero writes these; a scan reads them and never creates them
|
||||
write_default_configs(repo / ".openultrasast/learning/configs", load_families(), prompt="You hunt.", version="0")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted")
|
||||
return repo
|
||||
|
||||
|
||||
def _run_dir(repo: Path) -> Path:
|
||||
return sorted((repo / ".runs").iterdir())[-1]
|
||||
|
||||
|
||||
def _scan(repo: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
|
||||
assert main(["scan", str(repo), "--mode", "standard", "--config", str(repo / "openultrasast.toml")]) == 0
|
||||
run_dir = _run_dir(repo)
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
return findings, manifest
|
||||
|
||||
|
||||
def test_a_standard_scan_tags_findings_with_family_detector_and_verifier(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
|
||||
findings, manifest = _scan(repo)
|
||||
detected = [item for item in findings if any(str(tag).startswith("family:") for tag in item["tags"])]
|
||||
assert detected, "the family detectors ran and their findings are in the report"
|
||||
tags = set(detected[0]["tags"])
|
||||
assert any(tag.startswith("family:") for tag in tags)
|
||||
assert any(tag.startswith("detector:") for tag in tags)
|
||||
assert any(tag.startswith("verifier:") for tag in tags)
|
||||
assert detected[0]["evidence_level"] == "suspicion" # nothing rises without a verifier saying so
|
||||
block = manifest["learning"]
|
||||
assert block["taxonomy_version"] and block["round"] is None
|
||||
assert block["families"] and isinstance(block["classifier"], dict)
|
||||
assert block["classifier"]["hierarchical_credit"] is False
|
||||
|
||||
|
||||
def test_a_missing_family_directory_is_a_recorded_reason_not_a_crash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = _repo(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
'[learning]\nenabled = true\nconfigs_dir = "nowhere/configs"\n\n[models]\nhunter = "scripted-model"\n',
|
||||
configs=False,
|
||||
)
|
||||
findings, manifest = _scan(repo)
|
||||
assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"])
|
||||
reasons = {str(item.get("reason")) for item in manifest["degradations"]}
|
||||
assert "learning_configs_unavailable" in reasons
|
||||
assert "learning" not in manifest or manifest["learning"]["families"] == {}
|
||||
|
||||
|
||||
def test_the_feature_is_off_by_configuration_and_leaves_no_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = _repo(tmp_path, monkeypatch, "[learning]\nenabled = false\n", configs=False)
|
||||
findings, manifest = _scan(repo)
|
||||
assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"])
|
||||
assert "learning" not in manifest
|
||||
|
||||
|
||||
def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake")
|
||||
assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0
|
||||
run_dir = _run_dir(repo)
|
||||
verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
|
||||
payload = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
family_findings = {item["finding_id"] for item in payload if any(str(tag).startswith("family:") for tag in item["tags"])}
|
||||
assert family_findings, "the fixture must actually produce family findings, or this proves nothing"
|
||||
# A family detector's claim is a suspicion until a verifier speaks; handing it to the sandbox would let an
|
||||
# unverified model claim drive execution (Req 7.1).
|
||||
candidates = {str(item.get("finding_id") or item.get("id") or "") for item in verdicts}
|
||||
assert candidates.isdisjoint(family_findings)
|
||||
|
||||
|
||||
def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n')
|
||||
_scan(repo)
|
||||
sarif = json.loads((_run_dir(repo) / "report.sarif").read_text())
|
||||
props = [result["properties"] for result in sarif["runs"][0]["results"]]
|
||||
family_props = [item for item in props if item.get("family")]
|
||||
assert family_props and all(item.get("detector") and item.get("verifier") for item in family_props)
|
||||
@@ -1,185 +0,0 @@
|
||||
"""learning-harness task 2.3: per-pair family scoring (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
RANGES = {"app.py": (("leaky", 10, 13), ("guarded", 4, 7))}
|
||||
|
||||
|
||||
def _finding(family: str, line: int, path: str = "app.py") -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"detector:{path}:{line}",
|
||||
path=path,
|
||||
title="t",
|
||||
severity="medium",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name=None,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", "detector:x@1"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
def _case(tmp_path: Path, *, family: str = "access_control", unscorable: str | None = None) -> PairCase:
|
||||
(tmp_path / "v.py").write_text("x = 1\n")
|
||||
(tmp_path / "f.py").write_text("x = 2\n")
|
||||
return PairCase(
|
||||
name="p",
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / "v.py",
|
||||
fixed_file=tmp_path / "f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function="leaky", mechanism="other", family=family
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
unscorable=unscorable,
|
||||
)
|
||||
|
||||
|
||||
def test_detection_needs_the_labeled_family_inside_the_labeled_function(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path)
|
||||
hit = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert hit.outcome == "pair_correct" and hit.runs == ("pair_correct",)
|
||||
outside = score_pair_family(case, "access_control", [[_finding("access_control", 5)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert outside.outcome == "both_silent" # the right family in the wrong function is not a detection
|
||||
other_file = score_pair_family(
|
||||
case, "access_control", [[_finding("access_control", 12, "other.py")]], [[]], ranges=RANGES, taxonomy=taxonomy
|
||||
)
|
||||
assert other_file.outcome == "both_silent"
|
||||
wrong_family = score_pair_family(case, "access_control", [[_finding("injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert wrong_family.outcome == "both_silent" and wrong_family.other_findings_vuln == 1
|
||||
|
||||
|
||||
def test_a_fabricated_family_never_counts_and_is_recorded(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
score = score_pair_family(_case(tmp_path), "access_control", [[_finding("sql_injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert score.outcome == "both_silent" and score.fabricated_vuln == 1
|
||||
assert score.other_findings_vuln == 0 # a fabricated id is not another family, it is noise
|
||||
|
||||
|
||||
def test_silence_ignores_findings_of_other_families_and_never_calls_them_leaks(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path)
|
||||
quiet = score_pair_family(
|
||||
case,
|
||||
"access_control",
|
||||
[[_finding("access_control", 12)]],
|
||||
[[_finding("injection", 12), _finding("access_control", 5)]],
|
||||
ranges=RANGES,
|
||||
taxonomy=taxonomy,
|
||||
)
|
||||
assert quiet.outcome == "pair_correct" and quiet.other_findings_fixed == 2
|
||||
leaked = score_pair_family(
|
||||
case, "access_control", [[_finding("access_control", 12)]], [[_finding("access_control", 13)]], ranges=RANGES, taxonomy=taxonomy
|
||||
)
|
||||
assert leaked.outcome == "both_flagged"
|
||||
reversed_pair = score_pair_family(case, "access_control", [[]], [[_finding("access_control", 12)]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert reversed_pair.outcome == "reversed"
|
||||
|
||||
|
||||
def test_the_outcome_of_several_runs_is_their_majority(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path)
|
||||
hit = [_finding("access_control", 12)]
|
||||
score = score_pair_family(case, "access_control", [hit, [], hit], [[], [], []], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert score.runs == ("pair_correct", "both_silent", "pair_correct") and score.outcome == "pair_correct"
|
||||
flaky = score_pair_family(case, "access_control", [hit, [], []], [[], [], []], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert flaky.outcome == "both_silent" and flaky.flips == 1 # a run that disagrees with the majority is a flip
|
||||
|
||||
|
||||
def test_an_unscorable_pair_is_never_run_and_keeps_its_reason(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
case = _case(tmp_path, unscorable="identical_twin")
|
||||
score = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert score.outcome == "unscorable" and score.unscorable_reason == "identical_twin" and score.runs == ()
|
||||
|
||||
|
||||
def test_a_pair_with_no_parsed_function_range_is_unscorable_not_a_miss(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy)
|
||||
assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label"
|
||||
|
||||
|
||||
def test_scoring_never_reads_the_text_of_a_finding(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
shouting = _finding("injection", 12)
|
||||
loud = StaticFinding(**{**shouting.__dict__, "title": "CWE-639 broken object level authorization", "rationale": "IDOR in leaky"})
|
||||
score = score_pair_family(_case(tmp_path), "access_control", [[loud]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert score.outcome == "both_silent" # the words say access control; the family tag says injection, and only that counts
|
||||
|
||||
|
||||
def test_the_rung_is_the_best_verifier_outcome_among_the_detections(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
corroborated = _finding("access_control", 12)
|
||||
corroborated.tags.append("verifier:static_corroboration")
|
||||
score = score_pair_family(_case(tmp_path), "access_control", [[corroborated]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert score.outcome == "pair_correct" and score.rung == "static_corroboration"
|
||||
plain = score_pair_family(_case(tmp_path), "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy)
|
||||
assert plain.rung == "suspicion"
|
||||
|
||||
|
||||
def test_the_unscorable_reason_tells_a_tooling_gap_from_a_corpus_defect(tmp_path: Path) -> None:
|
||||
"""Req 4.1: an unparsable row is a tooling gap; a row whose labeled function no range names is a corpus defect."""
|
||||
from openultrasast.learning.scoring import unscorable_reason
|
||||
|
||||
case = _case(tmp_path)
|
||||
assert unscorable_reason(case, parse_ok=True, ranges=RANGES) is None
|
||||
assert unscorable_reason(case, parse_ok=False, ranges={}) == "unsupported_language"
|
||||
assert unscorable_reason(case, parse_ok=False, ranges=RANGES) == "unsupported_language" # the parse decides, not the ranges
|
||||
assert unscorable_reason(case, parse_ok=True, ranges={}) == "unresolved_label"
|
||||
assert unscorable_reason(_case(tmp_path, unscorable="identical_twin"), parse_ok=True, ranges=RANGES) == "identical_twin"
|
||||
twin = _case(tmp_path, unscorable="known_limit:java hash")
|
||||
assert unscorable_reason(twin, parse_ok=False, ranges={}) == "known_limit:java hash" # a declared limit wins
|
||||
|
||||
|
||||
def test_a_row_whose_registration_context_is_missing_is_named_as_such(tmp_path: Path) -> None:
|
||||
"""Req 4.1: an access-control row whose handler arrived without its registration cannot be scored for that reason."""
|
||||
from openultrasast.learning.scoring import unscorable_reason
|
||||
|
||||
case = _case(tmp_path, family="access_control")
|
||||
assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=("leaky",)) is None
|
||||
assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=()) == "missing_context"
|
||||
# only families whose bug shape needs the registration are judged on it
|
||||
assert unscorable_reason(_case(tmp_path, family="injection"), parse_ok=True, ranges=RANGES, entry_points=()) is None
|
||||
|
||||
|
||||
def test_an_unparsable_side_is_scored_unscorable_rather_than_missed(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.scoring import score_pair_family
|
||||
|
||||
taxonomy = load_families()
|
||||
score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy, parse_ok=False)
|
||||
assert score.outcome == "unscorable" and score.unscorable_reason == "unsupported_language"
|
||||
@@ -1,148 +0,0 @@
|
||||
"""learning-harness task 3.1: one teacher rule across every learning path (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored
|
||||
|
||||
VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
||||
FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n"
|
||||
|
||||
|
||||
def _teaching_case(tmp_path: Path, name: str, *, split: str = "train") -> PairCase:
|
||||
"""A pair that really carries a mechanism shape.
|
||||
|
||||
The trivial `x = 1` fixture below teaches nothing whatever the split says, so a teacher-rule assertion made
|
||||
against it passes with the rule deleted. This one is the difference between measuring the rule and measuring
|
||||
an empty corpus."""
|
||||
(tmp_path / f"{name}-v.py").write_text(VULN)
|
||||
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
split=split,
|
||||
review_tier="seeded",
|
||||
)
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase:
|
||||
(tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}.py",
|
||||
fixed_file=tmp_path / f"{name}.py",
|
||||
relpath="app.py",
|
||||
expected=(ExpectedFinding(cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", function="f", mechanism="other"),),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
split=split,
|
||||
review_tier=tier,
|
||||
reviewer="m" if tier == "reviewed" else "",
|
||||
unscorable=unscorable,
|
||||
)
|
||||
|
||||
|
||||
def test_only_vendored_trusted_scorable_train_pairs_may_teach(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.split import teachers
|
||||
|
||||
cases = [
|
||||
_case(tmp_path, "train-seeded"),
|
||||
_case(tmp_path, "train-reviewed", tier="reviewed"),
|
||||
_case(tmp_path, "holdout-seeded", split="holdout"),
|
||||
_case(tmp_path, "train-untrusted", tier="title"),
|
||||
_case(tmp_path, "train-twin", unscorable="identical_twin"),
|
||||
]
|
||||
assert [case.name for case in teachers(cases)] == ["train-seeded", "train-reviewed"]
|
||||
|
||||
|
||||
def test_a_candidate_taught_by_a_holdout_pair_is_refused_by_name(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.split import refuse_if_holdout
|
||||
|
||||
cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
|
||||
assert refuse_if_holdout(["train-one"], cases) is None
|
||||
refusal = refuse_if_holdout(["train-one", "holdout-one"], cases)
|
||||
assert refusal is not None and refusal.pairs == ("holdout-one",)
|
||||
assert refusal.reason == "holdout_pair_refused"
|
||||
degradation = refusal.degradation()
|
||||
assert degradation["stage"] == "learning" and degradation["pairs"] == ["holdout-one"]
|
||||
assert "holdout-one" in str(degradation["detail"])
|
||||
|
||||
|
||||
def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, corpus_mechanisms
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
cases = select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py"))
|
||||
holdout = {case.name for case in cases if case.split == "holdout"}
|
||||
store = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
report = export_mechanisms(cases, store)
|
||||
taught_by_holdout = [record.id for record in corpus_mechanisms(store.load()) if set(record.pairs) & holdout]
|
||||
assert taught_by_holdout == []
|
||||
assert report.records > 0 and any(reason.startswith("split:") for _pair, reason in report.skipped)
|
||||
|
||||
|
||||
def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None:
|
||||
"""Both arms, because `teaching_pairs <= 1` on a two-pair corpus is satisfied by a rule that does nothing.
|
||||
|
||||
The same two excerpts teach twice when both are on the train split and once when one is held out, and the run
|
||||
names the pair it refused (Req 5.1, 5.2)."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
both_train = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "train-two")])
|
||||
assert both_train.teaching_pairs == 2 and both_train.degradations == ()
|
||||
result = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")])
|
||||
assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"} # every pair is still scored
|
||||
assert result.teaching_pairs == 1
|
||||
refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
|
||||
assert [item["pairs"] for item in refusals] == [["holdout-one"]]
|
||||
|
||||
|
||||
def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None:
|
||||
"""Both arms. The shape is the one the corpus really teaches, taken from the pair itself.
|
||||
|
||||
Asserting only the refusal would pass against a lever that admits nothing at all, and a hand-written shape that
|
||||
matches no excerpt can never be admitted either, so neither would measure the split rule (Req 5.2)."""
|
||||
from openultrasast.improve.evolve import propose_mechanism_edits
|
||||
from openultrasast.semantic.loo import _facts
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.seed import pair_lessons
|
||||
|
||||
cases = [_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")]
|
||||
(lesson,) = pair_lessons(cases[0], _facts())
|
||||
|
||||
def store(name: str, pair: str) -> MechanismStore:
|
||||
path = MechanismStore(tmp_path / name)
|
||||
append_from_pair(path, lesson.shape, summary=lesson.summary, cwe=lesson.cwe, pair=pair, provenance="human", tier="seeded")
|
||||
return path
|
||||
|
||||
reasons: list[dict[str, object]] = []
|
||||
refused = propose_mechanism_edits(cases, store("refused.jsonl", "holdout-one"), tmp_path / "scan.jsonl", degradations=reasons)
|
||||
assert [edit.mechanism_id for edit in refused if edit.action == "admit"] == []
|
||||
assert [item["pairs"] for item in reasons] == [["holdout-one"]]
|
||||
admitted = propose_mechanism_edits(cases, store("allowed.jsonl", "train-one"), tmp_path / "scan2.jsonl")
|
||||
assert [edit.mechanism_id for edit in admitted if edit.action == "admit"], "a train-taught shape is still admissible"
|
||||
|
||||
|
||||
def test_the_corrected_lever_number_is_measured_and_committed() -> None:
|
||||
report = Path("benchmarks/measurements/2026-09-06-mechanism-lever-split.json")
|
||||
assert report.is_file()
|
||||
payload = json.loads(report.read_text())
|
||||
assert payload["slice"] == "vibe-py" and payload["holdout_pairs"] > 0
|
||||
assert set(payload["before"]) >= {"detected", "silent", "youden", "shapes"}
|
||||
assert set(payload["after"]) >= {"detected", "silent", "youden", "shapes"}
|
||||
assert payload["after"]["shapes"] < payload["before"]["shapes"] # holdout pairs no longer teach
|
||||
assert payload["after"]["youden"] <= payload["before"]["youden"]
|
||||
@@ -1,69 +0,0 @@
|
||||
"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3).
|
||||
|
||||
Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys
|
||||
determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did
|
||||
not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is
|
||||
built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.config import load_config
|
||||
|
||||
|
||||
def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None:
|
||||
"""The default preserves what round zero was measured under; changing it silently would make the two runs
|
||||
incomparable while looking like one number."""
|
||||
assert load_config(None).learning.thinking is False
|
||||
path = tmp_path / "openultrasast.toml"
|
||||
path.write_text("[learning]\nthinking = true\n")
|
||||
assert load_config(path).learning.thinking is True
|
||||
|
||||
|
||||
def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint
|
||||
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "k")
|
||||
monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False)
|
||||
path = tmp_path / "openultrasast.toml"
|
||||
path.write_text("[learning]\nthinking = true\n")
|
||||
client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc]
|
||||
assert endpoint.thinking is True
|
||||
assert client._disable_thinking is False # type: ignore[attr-defined]
|
||||
path.write_text("[learning]\nthinking = false\n")
|
||||
client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc]
|
||||
assert endpoint.thinking is False and client._disable_thinking is True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
|
||||
"""The condition that produced a number is part of what identifies it."""
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from test_learning_rounds import _always, _case, _steady # noqa: PLC0415
|
||||
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.learning.rounds import compare_baselines, run_baseline
|
||||
|
||||
out = tmp_path / "out"
|
||||
for thinking in (False, True):
|
||||
run_baseline(
|
||||
[_case(tmp_path, "a")],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan_factory=_always(_steady),
|
||||
model="deepseek-v4-flash",
|
||||
out_dir=out,
|
||||
k_runs=3,
|
||||
thinking=thinking,
|
||||
)
|
||||
written = sorted(path.name for path in (out / "baseline").iterdir())
|
||||
assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written
|
||||
assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"}
|
||||
report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text())
|
||||
assert report["thinking"] is True
|
||||
@@ -1,145 +0,0 @@
|
||||
"""learning-harness task 2.6: the verifier registry and the oracle callback (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result
|
||||
from openultrasast.sandbox import SandboxResult
|
||||
|
||||
APP = (
|
||||
"from flask import request\n\n\n"
|
||||
"@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
|
||||
" return Book.query.filter_by(book_title=title).first()\n\n\n"
|
||||
"@app.route('/books/any/<title>')\ndef leaky(title):\n"
|
||||
" owner = request.args.get('owner')\n"
|
||||
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
|
||||
"@app.route('/health')\ndef health():\n return {'ok': True}\n"
|
||||
)
|
||||
|
||||
|
||||
def _claim(function: str, line: int, family: str) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=f"detector:app.py:{line}",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="medium",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="r",
|
||||
line=line,
|
||||
function_name=function,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path: Path) -> Path:
|
||||
(tmp_path / "app.py").write_text(APP)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None:
|
||||
from openultrasast.learning.verifiers import token_oracle
|
||||
|
||||
oracle = token_oracle("CANARY:9f2a")
|
||||
fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False)
|
||||
quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False)
|
||||
assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE
|
||||
assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE # a crash is not this family's oracle
|
||||
assert verdict_from_result(quiet).verdict == TRIGGERABLE # without an oracle the exit code still decides
|
||||
assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired"
|
||||
|
||||
|
||||
def test_an_oracle_never_overrides_an_inconclusive_run() -> None:
|
||||
from openultrasast.learning.verifiers import token_oracle
|
||||
|
||||
oracle = token_oracle("CANARY:9f2a")
|
||||
timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True)
|
||||
assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive"
|
||||
assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive"
|
||||
assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable"
|
||||
|
||||
|
||||
def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None:
|
||||
from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for
|
||||
|
||||
taxonomy = load_families()
|
||||
assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier)
|
||||
assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier)
|
||||
assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier)
|
||||
# a canary family with no canary implementation registered yet reports at suspicion rather than pretending
|
||||
injection = verifier_for(taxonomy.by_id("injection"))
|
||||
assert isinstance(injection, NoVerifier) and injection.kind == "canary"
|
||||
|
||||
|
||||
def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None:
|
||||
from openultrasast.learning.verifiers import verifier_for
|
||||
|
||||
taxonomy = load_families()
|
||||
verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python")
|
||||
assert verification.rung == "suspicion" and verification.reason == "no_verifier"
|
||||
|
||||
|
||||
def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None:
|
||||
from openultrasast.learning.verifiers import verifier_for
|
||||
|
||||
taxonomy = load_families()
|
||||
verifier = verifier_for(taxonomy.by_id("access_control"))
|
||||
agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python")
|
||||
assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output
|
||||
# the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate
|
||||
healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python")
|
||||
assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees"
|
||||
elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python")
|
||||
assert elsewhere.rung == "suspicion"
|
||||
|
||||
|
||||
def test_a_verification_is_written_onto_the_claim_as_a_tag_and_nothing_else(repo: Path) -> None:
|
||||
"""The evidence ladder belongs to another boundary; a verifier says what it saw with a tag."""
|
||||
from openultrasast.learning.verifiers import apply_verification, verifier_for
|
||||
|
||||
taxonomy = load_families()
|
||||
claim = _claim("leaky", 12, "access_control")
|
||||
verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python"))
|
||||
assert "verifier:static_corroboration" in verified.tags
|
||||
assert verified.evidence_level == "suspicion" and claim.evidence_level == "suspicion" # the ladder is untouched
|
||||
plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python"))
|
||||
assert "verifier:suspicion" in plain.tags and plain.tags.count("verifier:suspicion") == 1
|
||||
|
||||
|
||||
def test_a_rung_only_reaches_the_ladder_through_an_explicit_mapping() -> None:
|
||||
"""Every rung must land on a level the project's ladder actually defines, or the report raises later."""
|
||||
from openultrasast.learning.verifiers import RUNGS, evidence_level_for
|
||||
from openultrasast.verification import EvidenceLevel
|
||||
|
||||
assert {rung: evidence_level_for(rung) for rung in RUNGS} == {
|
||||
"suspicion": EvidenceLevel.SUSPICION,
|
||||
"static_corroboration": EvidenceLevel.STATIC_CORROBORATION,
|
||||
"proven": EvidenceLevel.EXPLOIT_DEMONSTRATED,
|
||||
}
|
||||
assert all(EvidenceLevel(evidence_level_for(rung).value) for rung in RUNGS) # never a value outside the ladder
|
||||
|
||||
|
||||
def test_a_verified_claim_still_passes_the_projects_own_verification_helpers(repo: Path) -> None:
|
||||
"""The bug this pins: stamping a learning rung onto evidence_level raised ValueError in three call sites."""
|
||||
from openultrasast.learning.verifiers import Verification, apply_verification, raise_to
|
||||
from openultrasast.verification import VerificationStatus, is_report_verified, verify_finding
|
||||
|
||||
claim = _claim("lookup", 3, "injection")
|
||||
proven = apply_verification(claim, Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY x"))
|
||||
assert "verifier:proven" in proven.tags
|
||||
assert verify_finding(proven).evidence_level == "suspicion" # would raise ValueError if the rung were stamped on
|
||||
assert is_report_verified(proven.evidence_level, VerificationStatus.ACCEPTED) is False
|
||||
raised = raise_to(proven, "proven")
|
||||
assert raised.evidence_level == "exploit_demonstrated"
|
||||
assert verify_finding(raised).evidence_level == "exploit_demonstrated"
|
||||
assert is_report_verified(raised.evidence_level, VerificationStatus.ACCEPTED) is True
|
||||
@@ -1,235 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 3.1: leave-one-out is the corpus's own detection rate (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.pairs import PairCase
|
||||
|
||||
EXEC_A = (
|
||||
"import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n"
|
||||
+ " c.execute(\"SELECT * FROM t WHERE u = '%s'\" % user)\n"
|
||||
)
|
||||
EXEC_A_FIX = (
|
||||
"import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n c.execute('SELECT * FROM t WHERE u = ?', (user,))\n"
|
||||
)
|
||||
EXEC_B = (
|
||||
"import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n cur.execute('DELETE FROM t WHERE n = ' + name)\n"
|
||||
)
|
||||
EXEC_B_FIX = (
|
||||
"import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n"
|
||||
+ " cur.execute('DELETE FROM t WHERE n = ?', (name,))\n"
|
||||
)
|
||||
SYS_C = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n"
|
||||
SYS_C_FIX = "import os\nimport subprocess\n\n\ndef ping(host):\n subprocess.run(['ping', host])\n"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(vuln)
|
||||
(tmp_path / f"{name}-f.py").write_text(fixed)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="toy",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink"
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
provenance=provenance,
|
||||
review_tier="seeded",
|
||||
)
|
||||
|
||||
|
||||
def _toy(tmp_path: Path) -> list[PairCase]:
|
||||
return [
|
||||
_case(tmp_path, "a", EXEC_A, EXEC_A_FIX, function="lookup", cwe="CWE-89"),
|
||||
_case(tmp_path, "b", EXEC_B, EXEC_B_FIX, function="delete", cwe="CWE-89", provenance="agent"),
|
||||
_case(tmp_path, "c", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78"),
|
||||
]
|
||||
|
||||
|
||||
def test_leave_one_out_reports_which_pair_taught_which(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
result = evaluate_loo(_toy(tmp_path))
|
||||
by_name = {outcome.pair: outcome for outcome in result.outcomes}
|
||||
assert set(by_name) == {"a", "b", "c"}
|
||||
assert by_name["a"].detected and by_name["a"].silent and by_name["a"].found_by # taught by b's execute shape
|
||||
assert by_name["b"].detected and by_name["b"].silent and any("a" in pairs for _, pairs in by_name["b"].found_by)
|
||||
assert not by_name["c"].detected and by_name["c"].found_by == () # nobody else teaches os.system: c only tests
|
||||
slice_metrics = result.per_slice["toy"]
|
||||
assert slice_metrics["pairs"] == 3 and slice_metrics["detected"] == 2 and slice_metrics["silent"] == 3
|
||||
assert abs(slice_metrics["recall"] - 2 / 3) < 1e-9 and slice_metrics["silence"] == 1.0 and abs(slice_metrics["youden"] - 2 / 3) < 1e-9
|
||||
assert result.per_profile["agent"]["pairs"] == 1 and result.per_profile["human"]["pairs"] == 2
|
||||
assert "source_reaches_sink" in result.per_mechanism
|
||||
payload = result.to_dict()
|
||||
assert set(payload) >= {"per_slice", "per_profile", "per_mechanism", "outcomes"}
|
||||
json.dumps(payload) # artifact-ready
|
||||
|
||||
|
||||
def test_leave_one_out_seeds_only_trusted_pairs_and_skips_known_limit(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
cases = _toy(tmp_path)
|
||||
cases[1] = PairCase(**{**cases[1].__dict__, "review_tier": "advisory"}) # b may be tested, never teaches
|
||||
cases[2] = PairCase(**{**cases[2].__dict__, "known_limit": "engine"})
|
||||
result = evaluate_loo(cases)
|
||||
by_name = {outcome.pair: outcome for outcome in result.outcomes}
|
||||
assert set(by_name) == {"a", "b"} # known_limit excluded entirely
|
||||
assert not by_name["a"].detected # b is advisory: it cannot teach a
|
||||
assert by_name["b"].detected # a (seeded) teaches b, which is scored as a held-out target
|
||||
# `b` is excluded from teaching for its tier and says so; without that line a pair that silently stopped
|
||||
# teaching would look identical to one that never could (Req 5.2).
|
||||
assert result.skipped == (("c", "known_limit:engine"), ("b", "not_a_teacher:tier"))
|
||||
|
||||
|
||||
def test_cli_pairs_loo_writes_the_artifact_offline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
monkeypatch.chdir(Path.cwd())
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
assert main(["pairs", "--slice", "vibe-py", "--loo", "--json", "--loo-out", str(tmp_path / "loo.json")]) == 0
|
||||
payload = json.loads(buf.getvalue())
|
||||
# 35 vendored rows less the five byte-identical twins, which task 4.2 declared unscorable in the catalog: a
|
||||
# pair whose two sides are the same code cannot be held out against itself.
|
||||
assert payload["per_slice"]["vibe-py"]["pairs"] == 30 and "found_by" in payload["outcomes"][0]
|
||||
assert json.loads((tmp_path / "loo.json").read_text()) == payload
|
||||
|
||||
|
||||
HELPER_SINK = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n\n\ndef ping(host):\n return run('ping ' + host)\n"
|
||||
HELPER_SINK_FIX = (
|
||||
"import subprocess\n\n\ndef run(cmd):\n subprocess.run(cmd.split())\n\n\ndef ping(host):\n return run('ping ' + host)\n"
|
||||
)
|
||||
|
||||
|
||||
def test_hits_outside_the_labeled_function_or_off_label_never_count(tmp_path: Path) -> None:
|
||||
"""Round-3 mutants: a hit must sit inside the labeled function and name the labeled mechanism or sink."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
teacher = _case(tmp_path, "t", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78")
|
||||
outside = _case(
|
||||
tmp_path, "outside", HELPER_SINK, HELPER_SINK_FIX, function="ping", cwe="CWE-78"
|
||||
) # sink call lives in `run`, not `ping`
|
||||
off_label = PairCase(
|
||||
**{
|
||||
**_case(tmp_path, "offlabel", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78").__dict__,
|
||||
"expected": (
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="ping", mechanism="missing_auth_guard"
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
result = evaluate_loo([teacher, outside, off_label])
|
||||
by_name = {outcome.pair: outcome for outcome in result.outcomes}
|
||||
assert by_name["outside"].hits_vuln >= 1 and not by_name["outside"].detected
|
||||
assert by_name["offlabel"].hits_vuln >= 1 and not by_name["offlabel"].detected
|
||||
|
||||
|
||||
def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
|
||||
js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n"
|
||||
(tmp_path / "j-v.js").write_text(js)
|
||||
(tmp_path / "j-f.js").write_text(js)
|
||||
case = PairCase(
|
||||
**{
|
||||
**_toy(tmp_path)[0].__dict__,
|
||||
"name": "j",
|
||||
"language": "javascript",
|
||||
"vuln_file": tmp_path / "j-v.js",
|
||||
"fixed_file": tmp_path / "j-f.js",
|
||||
"relpath": "app.js",
|
||||
}
|
||||
)
|
||||
result = evaluate_loo([*_toy(tmp_path), case])
|
||||
assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations)
|
||||
|
||||
|
||||
# --- authorization-obligations 4.2: leave-one-out per obligation kind -------------------------------------------------
|
||||
|
||||
_HANDLER = "from flask import request\n\n\n@app.route('/{route}/<key>')\ndef {fn}(key):\n{body}"
|
||||
OBL_A = _HANDLER.format(route="books", fn="get_book", body=" return Book.query.filter_by(book_title=key).first()\n")
|
||||
OBL_A_FIX = _HANDLER.format(
|
||||
route="books",
|
||||
fn="get_book",
|
||||
body=" user_id = request.user.id\n return Book.query.filter_by(user_id=user_id, book_title=key).first()\n",
|
||||
)
|
||||
OBL_B = _HANDLER.format(route="orders", fn="get_order", body=" return Order.query.filter_by(ref=key).first()\n")
|
||||
OBL_B_FIX = _HANDLER.format(
|
||||
route="orders", fn="get_order", body=" owner = current_user.id\n return Order.query.filter_by(owner=owner, ref=key).first()\n"
|
||||
)
|
||||
OBL_C = _HANDLER.format(
|
||||
route="login", fn="login", body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='None')\n return resp\n"
|
||||
)
|
||||
OBL_C_FIX = _HANDLER.format(
|
||||
route="login",
|
||||
fn="login",
|
||||
body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='Strict', httponly=1)\n return resp\n",
|
||||
)
|
||||
|
||||
|
||||
def _obligation_case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, obligation: str, mechanism: str) -> PairCase:
|
||||
case = _case(tmp_path, name, vuln, fixed, function=function, cwe="CWE-639")
|
||||
row = case.expected[0]
|
||||
return PairCase(
|
||||
**{
|
||||
**case.__dict__,
|
||||
"slice": "vibe-py",
|
||||
"expected": (ExpectedFinding(**{**row.__dict__, "mechanism": mechanism, "obligation": obligation}),),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _obligation_toy(tmp_path: Path) -> list[PairCase]:
|
||||
read = {"obligation": "protected_read", "mechanism": "unconstrained_protected_read"}
|
||||
return [
|
||||
_obligation_case(tmp_path, "a", OBL_A, OBL_A_FIX, function="get_book", **read),
|
||||
_obligation_case(tmp_path, "b", OBL_B, OBL_B_FIX, function="get_order", **read),
|
||||
_obligation_case(tmp_path, "c", OBL_C, OBL_C_FIX, function="login", obligation="security_setting", mechanism="permissive_default"),
|
||||
]
|
||||
|
||||
|
||||
def test_leave_one_out_scores_obligation_rows_per_kind_and_names_the_teaching_pair(tmp_path: Path) -> None:
|
||||
"""Req 7.4: recall, silence and Youden per obligation kind; found_by names the shape record that supplied known_fix."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
result = evaluate_loo(_obligation_toy(tmp_path))
|
||||
by_name = {outcome.pair: outcome for outcome in result.outcomes}
|
||||
a, b, c = by_name["a"], by_name["b"], by_name["c"]
|
||||
assert a.obligations == ("protected_read",) and c.obligations == ("security_setting",)
|
||||
assert a.detected and a.silent and [pairs for _, pairs in a.found_by] == [("b",)] # b taught the identity discharger
|
||||
assert b.detected and b.silent and [pairs for _, pairs in b.found_by] == [("a",)]
|
||||
assert all(record_id.startswith("corpus:") for record_id, _ in a.found_by)
|
||||
assert c.teaches and not c.detected and c.found_by == () and c.silent # reached and undischarged, but nobody else teaches the kind
|
||||
kinds = result.per_obligation_kind
|
||||
assert kinds["protected_read"]["pairs"] == 2 and kinds["protected_read"]["detected"] == 2 and kinds["protected_read"]["youden"] == 1.0
|
||||
assert kinds["security_setting"]["pairs"] == 1 and kinds["security_setting"]["detected"] == 0
|
||||
assert "unconstrained_protected_read" in result.per_mechanism # flow mechanisms and obligation kinds sit side by side
|
||||
payload = result.to_dict()
|
||||
assert "per_obligation_kind" in payload and payload["outcomes"][0]["obligations"] == ["protected_read"]
|
||||
json.dumps(payload)
|
||||
|
||||
|
||||
def test_obligation_rows_are_not_found_by_the_facts_alone(tmp_path: Path) -> None:
|
||||
"""A finding without a known fix from the held-out store is the checker's own work, not the corpus teaching."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
(only,) = [outcome for outcome in evaluate_loo(_obligation_toy(tmp_path)[:1]).outcomes]
|
||||
assert only.hits_vuln >= 1 and not only.detected and only.found_by == () # the leaky read is flagged, but no pair taught it
|
||||
@@ -1,74 +0,0 @@
|
||||
"""corpus-seeded-mechanisms: additive store fields and the corpus writer (task 1.2, Req 1.3/1.4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_mechanism
|
||||
|
||||
|
||||
def _shape(**overrides: object):
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
base: dict[str, object] = {
|
||||
"language": "python",
|
||||
"sink_name": "system",
|
||||
"arity": 1,
|
||||
"source_positions": (0,),
|
||||
"source_kinds": ("fact_source",),
|
||||
"guard": "allowlist_test",
|
||||
"mechanism": "source_reaches_sink",
|
||||
}
|
||||
base.update(overrides)
|
||||
return Shape(**base) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_old_sandbox_rows_load_with_corpus_defaults(tmp_path: Path) -> None:
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
append_mechanism(store, summary="s", cwe="CWE-78", language="python", tags=("t",), what_made_it_exploitable="x")
|
||||
# a row written before this spec has no origin/tier/pairs/shape/guard fields at all
|
||||
raw = json.loads((tmp_path / "mechanisms.jsonl").read_text().splitlines()[0])
|
||||
for key in ("origin", "review_tier", "pairs", "shape", "guard"):
|
||||
raw.pop(key, None)
|
||||
(tmp_path / "mechanisms.jsonl").write_text(json.dumps(raw) + "\n")
|
||||
(record,) = store.load()
|
||||
assert (
|
||||
record.origin == "sandbox" and record.review_tier == "" and record.pairs == () and record.shape is None and record.guard == "none"
|
||||
)
|
||||
|
||||
|
||||
def test_append_from_pair_dedupes_by_shape_and_extends_provenance(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.mechanisms import append_from_pair, corpus_mechanisms
|
||||
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
first = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p1", provenance="human", tier="seeded")
|
||||
second = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p2", provenance="agent", tier="seeded")
|
||||
assert first.id == second.id
|
||||
assert (
|
||||
second.pairs == ("p1", "p2") and second.origin == "corpus" and second.review_tier == "seeded" and second.guard == "allowlist_test"
|
||||
)
|
||||
third = append_from_pair(
|
||||
store, _shape(guard="none"), summary="request into os.system", cwe="CWE-78", pair="p3", provenance="human", tier="reviewed"
|
||||
)
|
||||
assert third.id != first.id
|
||||
loaded = store.load()
|
||||
assert len(loaded) == 2 and {r.id for r in loaded} == {first.id, third.id} # the store is append-only, load folds by id
|
||||
corpus = corpus_mechanisms(loaded)
|
||||
assert {r.id for r in corpus} == {first.id, third.id}
|
||||
assert all(r.shape is not None and r.shape["sink_name"] == "system" for r in corpus)
|
||||
assert "source_reaches_sink" in first.tags and "mechanism:source_reaches_sink" in first.tags
|
||||
|
||||
|
||||
def test_corpus_rows_never_carry_source_text_and_ids_are_deterministic(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.mechanisms import append_from_pair
|
||||
|
||||
a = append_from_pair(
|
||||
MechanismStore(tmp_path / "a.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="p", provenance="human", tier="seeded"
|
||||
)
|
||||
b = append_from_pair(
|
||||
MechanismStore(tmp_path / "b.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="q", provenance="human", tier="seeded"
|
||||
)
|
||||
assert a.id == b.id # identity is the shape key, not a random uuid: the lever can name records across machines
|
||||
text = (tmp_path / "a.jsonl").read_text()
|
||||
assert "app.py" not in text and "cmd" not in text
|
||||
@@ -1,112 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 4.1: the `mechanisms` lever admits or retracts exporter records, nothing else."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
|
||||
def _record(store: MechanismStore, **overrides: object):
|
||||
base: dict[str, object] = {
|
||||
"language": "python",
|
||||
"sink_name": "system",
|
||||
"arity": 1,
|
||||
"source_positions": (0,),
|
||||
"source_kinds": ("parameter",),
|
||||
"guard": "allowlist_test",
|
||||
"mechanism": "source_reaches_sink",
|
||||
}
|
||||
base.update(overrides)
|
||||
return append_from_pair(store, Shape(**base), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_mechanism_edit_is_a_bounded_lever_with_a_stable_key(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.validator import VALID_LEVERS, MechanismEdit
|
||||
|
||||
assert "mechanisms" in VALID_LEVERS
|
||||
edit = MechanismEdit(action="admit", mechanism_id="corpus:abc", source="loo", rationale="recovers holdout pair x")
|
||||
assert edit.lever == "mechanisms" and edit.key() == "mechanisms:admit:corpus:abc"
|
||||
|
||||
|
||||
def test_validator_accepts_only_exporter_records_with_closed_guards(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError
|
||||
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
good = _record(candidates)
|
||||
validator = EvolveValidator()
|
||||
by_id = {record.id: record for record in candidates.load()}
|
||||
validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=good.id, source="loo"), by_id)
|
||||
validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=good.id, source="loo"), by_id)
|
||||
with pytest.raises(StrictValidationError, match="unknown mechanism"):
|
||||
validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id="corpus:not-exported", source="loo"), by_id)
|
||||
with pytest.raises(StrictValidationError, match="action"):
|
||||
validator.validate_mechanism(MechanismEdit(action="rewrite", mechanism_id=good.id, source="loo"), by_id)
|
||||
with pytest.raises(StrictValidationError, match="origin"):
|
||||
from openultrasast.semantic.mechanisms import append_mechanism
|
||||
|
||||
sandbox = append_mechanism(candidates, summary="s", cwe="CWE-78", language="python", tags=(), what_made_it_exploitable="x")
|
||||
validator.validate_mechanism(
|
||||
MechanismEdit(action="admit", mechanism_id=sandbox.id, source="loo"), {r.id: r for r in candidates.load()}
|
||||
)
|
||||
# free-form shapes never pass: a record whose shape text is not a closed structure is rejected before it can be admitted
|
||||
from dataclasses import replace
|
||||
|
||||
forged = replace(good, shape={**(good.shape or {}), "guard": "if user not in ALLOWED"})
|
||||
with pytest.raises(StrictValidationError, match="guard"):
|
||||
validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=forged.id, source="export"), {forged.id: forged})
|
||||
forged_text = replace(good, shape={**(good.shape or {}), "sink_name": "os.system(cmd + ' ')"})
|
||||
with pytest.raises(StrictValidationError, match="identifier"):
|
||||
validator.validate_mechanism(
|
||||
MechanismEdit(action="admit", mechanism_id=forged_text.id, source="export"), {forged_text.id: forged_text}
|
||||
)
|
||||
|
||||
|
||||
def test_apply_mechanism_edits_admits_into_the_scan_store_and_reverts_byte_for_byte(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits
|
||||
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
a = _record(candidates)
|
||||
b = _record(candidates, sink_name="popen")
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
scan_store.parent.mkdir()
|
||||
scan_store.write_text("")
|
||||
before = scan_store.read_bytes()
|
||||
snapshot = apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=a.id, source="loo")], candidates, scan_store)
|
||||
admitted = MechanismStore(scan_store).load()
|
||||
assert [r.id for r in admitted] == [a.id] and admitted[0].origin == "corpus"
|
||||
apply_mechanism_edits([MechanismEdit(action="retract", mechanism_id=a.id, source="loo")], candidates, scan_store)
|
||||
assert MechanismStore(scan_store).load() == () # retraction is a tombstone row; load() folds it away
|
||||
snapshot.restore()
|
||||
assert scan_store.read_bytes() == before # byte-for-byte revert of the whole round
|
||||
assert b.id != a.id
|
||||
|
||||
|
||||
def test_retract_is_allowed_for_an_admitted_record_that_left_the_candidate_set(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError
|
||||
|
||||
scan = MechanismStore(tmp_path / "scan.jsonl")
|
||||
stale = _record(scan, sink_name="popen")
|
||||
validator = EvolveValidator()
|
||||
with pytest.raises(StrictValidationError, match="unknown mechanism"):
|
||||
validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=stale.id), {}, admitted={stale.id: stale})
|
||||
validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=stale.id), {}, admitted={stale.id: stale})
|
||||
|
||||
|
||||
def test_export_defaults_to_a_candidates_file_and_never_writes_the_scan_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "benchmarks").symlink_to(Path(__file__).resolve().parents[1] / "benchmarks")
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
assert main(["mechanisms", "export", "--slice", "vibe-py", "--json"]) == 0
|
||||
payload = __import__("json").loads(buf.getvalue())
|
||||
assert payload["store"].endswith("mechanism-candidates.jsonl")
|
||||
assert not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists()
|
||||
@@ -1,330 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 4.2: the improve loop admits mechanisms that recover holdout pairs and retracts leakers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding, load_benchmark_manifest
|
||||
from openultrasast.pairs import PairCase
|
||||
from openultrasast.ruleset import PatternRule, write_ruleset
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
EXEC_VULN = (
|
||||
"import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n"
|
||||
+ " c.execute('SELECT * FROM t WHERE u = ' + user)\n"
|
||||
)
|
||||
EXEC_FIX = (
|
||||
"import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n"
|
||||
+ " c.execute('SELECT * FROM t WHERE u = ?', (user,))\n"
|
||||
)
|
||||
SYS_VULN = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n"
|
||||
SYS_LEAKY_FIX = (
|
||||
"import os\n\n\ndef ping(host):\n log(host)\n os.system('ping ' + host)\n" # the 'fix' still calls the sink: a leaking shape
|
||||
)
|
||||
|
||||
|
||||
def _shape(sink: str, guard: str = "none") -> Shape:
|
||||
return Shape(
|
||||
language="python",
|
||||
sink_name=sink,
|
||||
arity=1,
|
||||
source_positions=(0,),
|
||||
source_kinds=("parameter",),
|
||||
guard=guard,
|
||||
mechanism="source_reaches_sink",
|
||||
)
|
||||
|
||||
|
||||
def _pair(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(vuln)
|
||||
(tmp_path / f"{name}-f.py").write_text(fixed)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="github",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink"
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
provenance=provenance,
|
||||
split="holdout",
|
||||
review_tier="reviewed",
|
||||
reviewer="t",
|
||||
)
|
||||
|
||||
|
||||
def _round_env(tmp_path: Path) -> dict[str, object]:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\n")
|
||||
rd = tmp_path / "ruleset" / "python"
|
||||
write_ruleset(
|
||||
rd / "rules.toml",
|
||||
[PatternRule(rule_id="good-eval", title="e", languages=("python",), cwe="CWE-95", tags=("injection",), pattern=r"\beval\s*\(")],
|
||||
)
|
||||
manifest_path = tmp_path / "bench.toml"
|
||||
manifest_path.write_text(
|
||||
'name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n'
|
||||
'[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n'
|
||||
'line = 2\nrule_id = "good-eval"\nsink = "eval"\nevidence = "eval executes input"\n'
|
||||
)
|
||||
return {"target": repo, "manifest": load_benchmark_manifest(manifest_path), "ruleset_dir": tmp_path / "ruleset"}
|
||||
|
||||
|
||||
def test_proposer_admits_recovering_records_and_retracts_leakers(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import evaluate_mechanism_profiles, propose_mechanism_edits
|
||||
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
sys_record = append_from_pair(
|
||||
candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded"
|
||||
)
|
||||
holdout = [
|
||||
_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"),
|
||||
_pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78", provenance="agent"),
|
||||
]
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
edits = propose_mechanism_edits(holdout, candidates, scan_store)
|
||||
assert [(e.action, e.mechanism_id) for e in edits] == [("admit", exec_record.id)] # sys leaks on y's fixed side: never proposed
|
||||
assert "x" in edits[0].rationale
|
||||
# once a leaking record is admitted (by a human or an earlier round), the proposer asks to retract it
|
||||
from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits
|
||||
|
||||
apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=sys_record.id)], candidates, scan_store)
|
||||
edits = propose_mechanism_edits(holdout, candidates, scan_store)
|
||||
assert ("retract", sys_record.id) in [(e.action, e.mechanism_id) for e in edits]
|
||||
profiles = evaluate_mechanism_profiles(holdout, scan_store)
|
||||
assert profiles["agent"]["pairs"] == 1.0 and profiles["agent"]["pair_correct"] == 0.0 # y: detected but leaked
|
||||
|
||||
|
||||
def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for_byte(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import run_round
|
||||
from openultrasast.improve.validator import MechanismEdit
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
sys_record = append_from_pair(
|
||||
candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded"
|
||||
)
|
||||
holdout = [
|
||||
_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"),
|
||||
_pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78"),
|
||||
]
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
journal = tmp_path / "journal.json"
|
||||
outcome = run_round(
|
||||
env["target"],
|
||||
env["manifest"],
|
||||
ledger_path=ledger,
|
||||
journal_path=journal,
|
||||
ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type]
|
||||
pair_cases=holdout,
|
||||
min_holdout_pairs=1,
|
||||
mechanism_candidates=candidates.log_path,
|
||||
mechanism_store=scan_store,
|
||||
)
|
||||
assert outcome.accepted and outcome.reason == "accepted"
|
||||
assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id]
|
||||
entry = json.loads(journal.read_text())[0]
|
||||
mech_edits = [e for e in entry["edits"] if e["lever"] == "mechanisms"]
|
||||
assert mech_edits and mech_edits[0]["mechanism_id"] == exec_record.id and mech_edits[0]["pairs"] == ["teacher-a"]
|
||||
assert entry["mechanisms_before"]["human"]["pair_correct"] == 0.0 and entry["mechanisms_after"]["human"]["pair_correct"] == 1.0
|
||||
before = scan_store.read_bytes()
|
||||
forced = run_round(
|
||||
env["target"],
|
||||
env["manifest"],
|
||||
ledger_path=ledger,
|
||||
journal_path=journal,
|
||||
ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type]
|
||||
pair_cases=holdout,
|
||||
min_holdout_pairs=1,
|
||||
mechanism_candidates=candidates.log_path,
|
||||
mechanism_store=scan_store,
|
||||
scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")],
|
||||
)
|
||||
assert not forced.accepted and (forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression"))
|
||||
assert scan_store.read_bytes() == before # byte-for-byte revert
|
||||
|
||||
|
||||
def test_no_gain_admission_is_rejected_and_reverted_mechanism_edits_are_not_reproposed(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import run_round
|
||||
from openultrasast.improve.journal import reverted_edit_keys
|
||||
from openultrasast.improve.validator import MechanismEdit
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
unrelated = append_from_pair(
|
||||
candidates, _shape("popen"), summary="s", cwe="CWE-78", pair="teacher-c", provenance="human", tier="seeded"
|
||||
)
|
||||
holdout = [_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89")]
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
journal = tmp_path / "journal.json"
|
||||
kwargs: dict[str, object] = {
|
||||
"ledger_path": ledger,
|
||||
"journal_path": journal,
|
||||
"ruleset_dir": env["ruleset_dir"],
|
||||
"pair_cases": holdout,
|
||||
"min_holdout_pairs": 1,
|
||||
"mechanism_candidates": candidates.log_path,
|
||||
"mechanism_store": scan_store,
|
||||
}
|
||||
edit = MechanismEdit(action="admit", mechanism_id=unrelated.id, source="export", rationale="scripted: recovers nothing")
|
||||
outcome = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type]
|
||||
assert not outcome.accepted and outcome.reason == "mechanism_no_gain"
|
||||
assert not scan_store.exists() or scan_store.read_bytes() == b""
|
||||
assert edit.key() in reverted_edit_keys(__import__("json").loads(journal.read_text()))
|
||||
# the same reverted edit, offered again with a rationale, is filtered before validation (novelty gate)
|
||||
again = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type]
|
||||
assert again.reason == "no_proposals" and again.mechanism_edits == []
|
||||
|
||||
|
||||
def _holdout_catalog(tmp_path: Path) -> Path:
|
||||
"""A train pair to teach and a holdout pair to recover.
|
||||
|
||||
The catalog used to hold the holdout pair alone, so the only shape available was the one that pair
|
||||
taught about itself; learning-harness Req 5.1 refuses that, and rightly.
|
||||
"""
|
||||
(tmp_path / "x-v.py").write_text(EXEC_VULN)
|
||||
(tmp_path / "x-f.py").write_text(EXEC_FIX)
|
||||
(tmp_path / "t-v.py").write_text(EXEC_VULN.replace("lookup", "search").replace("u = ", "name = "))
|
||||
(tmp_path / "t-f.py").write_text(EXEC_FIX.replace("lookup", "search").replace("u = ", "name = "))
|
||||
row = (
|
||||
'[[pair]]\nname = "{name}"\nslice = "github"\nlanguage = "python"\nvuln = "{name[0]}-v.py"\nfixed = "{name[0]}-f.py"\n'
|
||||
'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "reviewed"\nreviewer = "t"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-89"\nclass = "sql injection"\npath = "app.py"\nfunction = "{function}"\n'
|
||||
'sink = "execute"\nmechanism = "source_reaches_sink"\n'
|
||||
)
|
||||
catalog = tmp_path / "pairs.toml"
|
||||
catalog.write_text(
|
||||
row.replace("{name}", "x").replace("{name[0]}", "x").replace("{split}", "holdout").replace("{function}", "lookup")
|
||||
+ "\n"
|
||||
+ row.replace("{name}", "t").replace("{name[0]}", "t").replace("{split}", "train").replace("{function}", "search")
|
||||
)
|
||||
return catalog
|
||||
|
||||
|
||||
def test_ousast_improve_admits_candidates_into_the_scan_store_by_default(tmp_path: Path) -> None:
|
||||
"""Req 5: the lever is reachable from the operator command; export -> candidates -> improve -> mechanisms.jsonl the scan reads."""
|
||||
from openultrasast.cli import main
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
repo = Path(str(env["target"]))
|
||||
calibration = repo / ".openultrasast" / "calibration"
|
||||
candidates = MechanismStore(calibration / "mechanism-candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
catalog = _holdout_catalog(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
assert (
|
||||
main(
|
||||
["improve", str(manifest), "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
admitted = MechanismStore(calibration / "mechanisms.jsonl").load()
|
||||
assert [r.id for r in admitted] == [exec_record.id]
|
||||
journal = json.loads((calibration / "improve_journal.json").read_text())
|
||||
assert any(e["lever"] == "mechanisms" and e["mechanism_id"] == exec_record.id for entry in journal for e in entry["edits"])
|
||||
|
||||
|
||||
def test_run_improvement_threads_mechanism_paths(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import run_improvement
|
||||
from openultrasast.pairs import load_pair_catalog
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
cases = load_pair_catalog(_holdout_catalog(tmp_path))
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
outcomes = run_improvement(
|
||||
env["target"],
|
||||
env["manifest"],
|
||||
ledger_path=tmp_path / "ledger.json",
|
||||
journal_path=tmp_path / "journal.json", # type: ignore[arg-type]
|
||||
ruleset_dir=env["ruleset_dir"],
|
||||
pair_cases=cases,
|
||||
min_holdout_pairs=1, # type: ignore[arg-type]
|
||||
mechanism_candidates=candidates.log_path,
|
||||
mechanism_store=scan_store,
|
||||
max_rounds=3,
|
||||
)
|
||||
assert any(o.accepted and o.mechanism_edits for o in outcomes)
|
||||
assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id]
|
||||
assert outcomes[-1].reason == "no_proposals" # converges once the candidate is admitted
|
||||
|
||||
|
||||
def _cli_env(tmp_path: Path) -> tuple[dict[str, object], Path, Path]:
|
||||
env = _round_env(tmp_path)
|
||||
calibration = Path(str(env["target"])) / ".openultrasast" / "calibration"
|
||||
candidates = MechanismStore(calibration / "mechanism-candidates.jsonl")
|
||||
append_from_pair(candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded")
|
||||
return env, calibration, _holdout_catalog(tmp_path)
|
||||
|
||||
|
||||
def test_dry_run_and_no_mechanisms_never_write_the_scan_store(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
from openultrasast.cli import main
|
||||
|
||||
env, calibration, catalog = _cli_env(tmp_path)
|
||||
base = [
|
||||
"improve",
|
||||
str(tmp_path / "bench.toml"),
|
||||
"--ruleset-dir",
|
||||
str(env["ruleset_dir"]),
|
||||
"--pair-catalog",
|
||||
str(catalog),
|
||||
"--min-holdout-pairs",
|
||||
"1",
|
||||
]
|
||||
assert main([*base, "--dry-run"]) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists() # a dry run mutates nothing under the target
|
||||
assert main([*base, "--no-mechanisms"]) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists()
|
||||
(calibration / "mechanism-candidates.jsonl").unlink()
|
||||
capsys.readouterr() # the dry run legitimately reports its scratch admission; judge only the candidate-less round below
|
||||
assert main(base) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists() # no candidates file: the lever is inert, the round is a plain rule round
|
||||
out = capsys.readouterr().out
|
||||
assert "admit " not in out and "retract " not in out
|
||||
|
||||
|
||||
def test_documented_default_flow_connects_export_to_improve(tmp_path: Path, monkeypatch, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
"""README flow: `ousast mechanisms export` (cwd default) then `ousast improve --pair-catalog` admits into the target's store."""
|
||||
from openultrasast.cli import main
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
catalog = _holdout_catalog(tmp_path)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# export a candidates file exactly as documented (cwd default), seeded from a tiny trusted catalog of our own
|
||||
assert main(["mechanisms", "export", "--catalog", str(catalog), "--slice", "all"]) == 0
|
||||
exported = tmp_path / ".openultrasast" / "calibration" / "mechanism-candidates.jsonl"
|
||||
assert exported.is_file() and not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists()
|
||||
# the target is a different directory (the benchmark fixture); improve must still find the cwd candidates by default
|
||||
assert (
|
||||
main(
|
||||
["improve", "bench.toml", "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
target_store = Path(str(env["target"])) / ".openultrasast" / "calibration" / "mechanisms.jsonl"
|
||||
assert MechanismStore(target_store).load() # admitted where the scan of that target reads
|
||||
out = capsys.readouterr().out
|
||||
assert "admit corpus:" in out # the accepted round names its mechanism edits
|
||||
@@ -1,141 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 5.1: reports name the mechanism, the known fix guard, and the pair provenance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.reports import write_markdown_report, write_sarif_report
|
||||
from openultrasast.semantic.overlay import OverlayRecord
|
||||
|
||||
|
||||
def _variant() -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="variant:corpus:abc:app.py:5",
|
||||
path="app.py",
|
||||
title="Variant of known mechanism: run_shell",
|
||||
severity="medium",
|
||||
confidence="low",
|
||||
evidence_level="suspicion",
|
||||
rationale="parameter into run_shell; fix adds allowlist_test. Structural variant of a mechanism learned from p1, p2.",
|
||||
line=5,
|
||||
function_name="handler",
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["variant", "mechanism:corpus:abc"],
|
||||
ranking_priority=0.0,
|
||||
)
|
||||
|
||||
|
||||
MECHANISMS = {
|
||||
"corpus:abc": {
|
||||
"summary": "source_reaches_sink: parameter into run_shell (CWE-78); fix adds allowlist_test",
|
||||
"guard": "allowlist_test",
|
||||
"pairs": ["p1", "p2"],
|
||||
"cwe": "CWE-78",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_markdown_names_mechanism_guard_and_provenance_and_keeps_the_suspicion_label(tmp_path: Path) -> None:
|
||||
path = tmp_path / "report.md"
|
||||
write_markdown_report([_variant()], path, mechanisms=MECHANISMS)
|
||||
text = path.read_text()
|
||||
assert "- Mechanism: `corpus:abc`" in text
|
||||
assert "source_reaches_sink: parameter into run_shell" in text
|
||||
assert "- Known fix guard: `allowlist_test`" in text
|
||||
assert "- Learned from pairs: `p1, p2`" in text
|
||||
assert "- Evidence level: `suspicion`" in text
|
||||
assert "## Mechanisms" in text # a section summarising every mechanism the report cites
|
||||
|
||||
|
||||
def test_sarif_carries_mechanism_properties_for_variant_and_merged_overlay_findings(tmp_path: Path) -> None:
|
||||
merged = OverlayRecord(
|
||||
proposal_id="python-os-command:app.py:9",
|
||||
path="app.py",
|
||||
line=9,
|
||||
disposition="promote",
|
||||
reason="request reaches os.system",
|
||||
cwe="CWE-78",
|
||||
sources=("request",),
|
||||
sinks=("os.system",),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
origin="inventory",
|
||||
engine="python-ast",
|
||||
language="python",
|
||||
mechanism_id="corpus:abc",
|
||||
)
|
||||
inventory = StaticFinding(
|
||||
**{
|
||||
**_variant().__dict__,
|
||||
"finding_id": "python-os-command:app.py:9",
|
||||
"line": 9,
|
||||
"tags": ["injection"],
|
||||
"evidence_level": "static_corroboration",
|
||||
}
|
||||
)
|
||||
path = tmp_path / "report.sarif"
|
||||
write_sarif_report([_variant(), inventory], [], path, overlay=[merged], mechanisms=MECHANISMS)
|
||||
results = json.loads(path.read_text())["runs"][0]["results"]
|
||||
by_id = {item["properties"]["finding_id"]: item["properties"] for item in results}
|
||||
variant = by_id["variant:corpus:abc:app.py:5"]
|
||||
assert (
|
||||
variant["mechanism_id"] == "corpus:abc"
|
||||
and variant["mechanism_guard"] == "allowlist_test"
|
||||
and variant["mechanism_pairs"] == ["p1", "p2"]
|
||||
)
|
||||
assert variant["evidence_level"] == "suspicion"
|
||||
merged_props = by_id["python-os-command:app.py:9"]
|
||||
assert merged_props["mechanism_id"] == "corpus:abc" and merged_props["mechanism_guard"] == "allowlist_test" # from the overlay record
|
||||
|
||||
|
||||
def test_reports_without_mechanisms_are_unchanged(tmp_path: Path) -> None:
|
||||
plain = StaticFinding(**{**_variant().__dict__, "finding_id": "python-os-command:app.py:9", "tags": ["injection"]})
|
||||
path = tmp_path / "report.md"
|
||||
write_markdown_report([plain], path)
|
||||
text = path.read_text()
|
||||
assert "Mechanism:" not in text and "## Mechanisms" not in text
|
||||
|
||||
|
||||
def test_standard_scan_report_names_the_mechanism_behind_a_variant_finding(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n")
|
||||
shape = Shape(
|
||||
language="python",
|
||||
sink_name="run_shell",
|
||||
arity=1,
|
||||
source_positions=(0,),
|
||||
source_kinds=("parameter",),
|
||||
guard="allowlist_test",
|
||||
mechanism="source_reaches_sink",
|
||||
)
|
||||
record = append_from_pair(
|
||||
MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"),
|
||||
shape,
|
||||
summary="parameter into run_shell",
|
||||
cwe="CWE-78",
|
||||
pair="p1",
|
||||
provenance="human",
|
||||
tier="seeded",
|
||||
)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
report = (run_dir / "report.md").read_text()
|
||||
assert (
|
||||
f"- Mechanism: `{record.id}`" in report
|
||||
and "- Known fix guard: `allowlist_test`" in report
|
||||
and "- Learned from pairs: `p1`" in report
|
||||
)
|
||||
sarif = json.loads((run_dir / "report.sarif").read_text())
|
||||
props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"]["finding_id"].startswith("variant:")]
|
||||
assert props and props[0]["mechanism_guard"] == "allowlist_test" and props[0]["evidence_level"] == "suspicion"
|
||||
@@ -1,112 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 2.1: export trusted pairs as mechanism records (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.pairs import PairCase, load_pair_catalog, select_slice
|
||||
from openultrasast.semantic.mechanisms import MechanismStore
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n"
|
||||
FIXED = (
|
||||
"from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n"
|
||||
" if cmd not in ALLOWED:\n return None\n os.system(cmd)\n"
|
||||
)
|
||||
|
||||
|
||||
def _case(tmp_path: Path, name: str, *, tier: str, known_limit: str | None = None, vuln: str = VULN, language: str = "python") -> PairCase:
|
||||
ext = {"python": "py", "javascript": "js"}[language]
|
||||
(tmp_path / f"{name}-v.{ext}").write_text(vuln)
|
||||
(tmp_path / f"{name}-f.{ext}").write_text(FIXED if language == "python" else vuln)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language=language,
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.{ext}",
|
||||
fixed_file=tmp_path / f"{name}-f.{ext}",
|
||||
relpath=f"app.{ext}",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78",
|
||||
vulnerability_class="command injection",
|
||||
path=f"app.{ext}",
|
||||
evidence="",
|
||||
sink="os.system",
|
||||
function="run",
|
||||
mechanism="source_reaches_sink",
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
provenance="human",
|
||||
review_tier=tier,
|
||||
reviewer="t" if tier == "reviewed" else "",
|
||||
known_limit=known_limit,
|
||||
)
|
||||
|
||||
|
||||
def test_export_seeds_only_trusted_parsing_pairs_and_reports_every_skip(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
cases = [
|
||||
_case(tmp_path, "seeded", tier="seeded"),
|
||||
_case(tmp_path, "reviewed-twin", tier="reviewed"),
|
||||
_case(tmp_path, "advisory", tier="advisory"),
|
||||
_case(tmp_path, "title", tier="title"),
|
||||
_case(tmp_path, "limit", tier="seeded", known_limit="engine cannot see it"),
|
||||
_case(tmp_path, "broken", tier="seeded", vuln="def run(:\n"),
|
||||
]
|
||||
report = export_mechanisms(cases, store)
|
||||
records = store.load()
|
||||
assert len(records) == 1 # seeded and reviewed-twin share one shape: one record, two pairs
|
||||
(record,) = records
|
||||
assert record.origin == "corpus" and record.pairs == ("seeded", "reviewed-twin") and record.shape is not None
|
||||
assert report.seeded == 2 and report.records == 1
|
||||
reasons = dict(report.skipped)
|
||||
assert reasons["advisory"] == "tier:advisory" and reasons["title"] == "tier:title"
|
||||
assert reasons["limit"].startswith("known_limit") and reasons["broken"] == "parse_failed"
|
||||
text = (tmp_path / "mechanisms.jsonl").read_text()
|
||||
assert "request.args" not in text and "app.py" not in text # Req 2.4: no code text in the store
|
||||
|
||||
|
||||
def test_export_skips_languages_without_a_grammar_with_a_degradation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n"
|
||||
report = export_mechanisms([_case(tmp_path, "js", tier="seeded", vuln=js, language="javascript")], store)
|
||||
assert store.load() == () and dict(report.skipped)["js"] == "variants_language_unsupported"
|
||||
assert {"stage": "mechanisms", "reason": "variants_language_unsupported", "language": "javascript"} in report.degradations
|
||||
|
||||
|
||||
def test_export_over_the_vendored_vibe_py_slice_is_offline_and_deterministic(tmp_path: Path) -> None:
|
||||
from openultrasast.pairs import select_vendored
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py"))
|
||||
first = export_mechanisms(cases, MechanismStore(tmp_path / "a.jsonl"))
|
||||
second = export_mechanisms(cases, MechanismStore(tmp_path / "b.jsonl"))
|
||||
assert first.records >= 1 and first.records == second.records
|
||||
assert (tmp_path / "a.jsonl").read_text() == (tmp_path / "b.jsonl").read_text()
|
||||
assert first.seeded + len(first.skipped) == len(cases)
|
||||
|
||||
|
||||
def test_cli_mechanisms_export_writes_the_store_and_prints_the_report(tmp_path: Path) -> None:
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
assert main(["mechanisms", "export", "--slice", "vibe-py", "--store", str(tmp_path / "m.jsonl"), "--json"]) == 0
|
||||
payload = json.loads(buf.getvalue())
|
||||
assert payload["records"] >= 1 and payload["slice"] == "vibe-py" and "skipped" in payload
|
||||
assert (tmp_path / "m.jsonl").is_file()
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -33,7 +34,7 @@ def _endpoint_env(monkeypatch: pytest.MonkeyPatch, **values: str | None) -> None
|
||||
|
||||
|
||||
def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint
|
||||
from openultrasast.model.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint
|
||||
|
||||
_endpoint_env(monkeypatch, DEEPSEEK_API_KEY="ds-key", OPENROUTER_API_KEY="or-key", OPENROUTER_BASE_URL="https://openrouter.ai/api/v1")
|
||||
resolved = resolve_chat_endpoint(ResolvedConfig())
|
||||
@@ -47,7 +48,7 @@ def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint(
|
||||
|
||||
|
||||
def test_resolution_order_override_then_scripted_then_openrouter_then_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint
|
||||
from openultrasast.model.endpoint import resolve_chat_endpoint
|
||||
from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient
|
||||
|
||||
injected = ScriptedChatClient([ChatResponse(content="{}")])
|
||||
@@ -68,7 +69,7 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke
|
||||
|
||||
|
||||
def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint
|
||||
from openultrasast.model.endpoint import resolve_chat_endpoint
|
||||
|
||||
_endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key")
|
||||
|
||||
@@ -81,16 +82,14 @@ def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: p
|
||||
|
||||
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
from openultrasast.config import load_config
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
|
||||
from openultrasast.model.endpoint import resolve_chat_endpoint, resolve_models
|
||||
|
||||
_endpoint_env(monkeypatch, MY_KEY="local-key")
|
||||
monkeypatch.setenv("MY_KEY", "local-key")
|
||||
(tmp_path / "ousast.toml").write_text(
|
||||
'[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n\n'
|
||||
"[learning]\nk_runs = 3\nround_cost_cap_usd = 2.5\nminibatch = 4\n"
|
||||
'[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n'
|
||||
)
|
||||
config = load_config(tmp_path / "ousast.toml")
|
||||
assert config.learning.k_runs == 3 and config.learning.round_cost_cap_usd == 2.5 and config.learning.minibatch == 4
|
||||
resolved = resolve_chat_endpoint(config)
|
||||
assert resolved is not None and resolved[1].base_url == "http://localhost:11434/v1"
|
||||
detector, judge = resolve_models(config)
|
||||
@@ -98,7 +97,7 @@ def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pyte
|
||||
|
||||
|
||||
def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_field() -> None:
|
||||
from openultrasast.learning.endpoint import DeepSeekChatClient
|
||||
from openultrasast.model.endpoint import DeepSeekChatClient
|
||||
|
||||
recorder = _Recorder([{"content": '{"intent": "public"}', "reasoning_content": "step one"}])
|
||||
client = DeepSeekChatClient(recorder) # type: ignore[arg-type]
|
||||
@@ -110,7 +109,7 @@ def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_fie
|
||||
|
||||
|
||||
def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None:
|
||||
from openultrasast.learning.endpoint import DeepSeekChatClient
|
||||
from openultrasast.model.endpoint import DeepSeekChatClient
|
||||
|
||||
recorder = _Recorder([{"content": ""}, {"content": '{"ok": true}'}])
|
||||
client = DeepSeekChatClient(recorder) # type: ignore[arg-type]
|
||||
@@ -123,7 +122,7 @@ def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None:
|
||||
|
||||
|
||||
def test_log_probabilities_are_requested_only_when_asked_for() -> None:
|
||||
from openultrasast.learning.endpoint import DeepSeekChatClient
|
||||
from openultrasast.model.endpoint import DeepSeekChatClient
|
||||
|
||||
recorder = _Recorder([{"content": "{}", "logprobs": {"content": [{"logprob": -0.1}, {"logprob": -0.3}]}}])
|
||||
client = DeepSeekChatClient(recorder) # type: ignore[arg-type]
|
||||
@@ -133,7 +132,7 @@ def test_log_probabilities_are_requested_only_when_asked_for() -> None:
|
||||
|
||||
|
||||
def test_cost_comes_from_the_providers_own_usage_fields() -> None:
|
||||
from openultrasast.learning.endpoint import ChatEndpoint, price_of
|
||||
from openultrasast.model.endpoint import ChatEndpoint, price_of
|
||||
|
||||
flash = ChatEndpoint(provider="deepseek", base_url="https://api.deepseek.com", thinking=False, prices=price_of("deepseek-v4-flash"))
|
||||
cost = flash.cost({"prompt_cache_hit_tokens": 1_000_000, "prompt_cache_miss_tokens": 1_000_000, "completion_tokens": 1_000_000})
|
||||
@@ -179,7 +178,7 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None
|
||||
|
||||
`_message_of` returned only `choices[0].message`, so the confidence signal the classifier is supposed to
|
||||
abstain on was always `None` — a knob that reads as "never unsure" whatever the model said."""
|
||||
from openultrasast.learning.endpoint import _message_of
|
||||
from openultrasast.model.endpoint import _message_of
|
||||
|
||||
payload = {
|
||||
"choices": [
|
||||
@@ -197,8 +196,29 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None
|
||||
|
||||
|
||||
def test_a_reply_without_logprobs_reports_none_rather_than_a_number() -> None:
|
||||
from openultrasast.learning.endpoint import _message_of
|
||||
from openultrasast.model.endpoint import _message_of
|
||||
from openultrasast.tool_hunter import chat_response_from_message
|
||||
|
||||
message = _message_of({"choices": [{"message": {"content": "yes"}}]})
|
||||
assert chat_response_from_message(message).mean_logprob is None
|
||||
|
||||
|
||||
def test_thinking_is_a_measured_condition_on_the_models_section() -> None:
|
||||
"""The knob moved from `[learning]` to `[models]` with the noise architecture's removal.
|
||||
|
||||
Thinking is off by default because that is what the committed baseline was measured under — the provider
|
||||
silently ignores `temperature` while thinking — not because it is known to be better.
|
||||
"""
|
||||
from openultrasast.config import load_config
|
||||
|
||||
assert load_config(None).models.thinking is False
|
||||
|
||||
|
||||
def test_the_config_can_turn_thinking_on(tmp_path: Path) -> None:
|
||||
from openultrasast.config import load_config
|
||||
|
||||
path = tmp_path / "ousast.toml"
|
||||
path.write_text("[models]\nthinking = true\n")
|
||||
assert load_config(path).models.thinking is True
|
||||
path.write_text("[models]\nthinking = false\n")
|
||||
assert load_config(path).models.thinking is False
|
||||
@@ -0,0 +1,101 @@
|
||||
"""model-grounded-detection task 1.1: the security vocabularies survive the modules that carried them.
|
||||
|
||||
The guard vocabulary lived in `semantic/variants.py`, whose optimisation loop this feature deletes. The
|
||||
source/sink/sanitizer facts and the obligation facts live in `ruleset/` data and are retained, so the specs
|
||||
*derive* from them rather than copying them — a copy would be a second place to update. What must be ported
|
||||
verbatim is the guard vocabulary, because nothing else holds it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
SPECS = Path("src/openultrasast/model/specs.py")
|
||||
|
||||
|
||||
def test_the_guard_vocabulary_is_preserved_verbatim() -> None:
|
||||
from openultrasast.model.specs import GUARD_KINDS
|
||||
|
||||
assert GUARD_KINDS == ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none")
|
||||
|
||||
|
||||
def test_the_guard_classifier_still_recognises_each_kind() -> None:
|
||||
"""The patterns classify the *fix*, first match wins — the order is part of the vocabulary."""
|
||||
from openultrasast.model.specs import classify_guard_text
|
||||
|
||||
assert classify_guard_text('cur.execute("SELECT * FROM t WHERE id = %s", (uid,))') == "parameterized_call"
|
||||
assert classify_guard_text("if not current_user.is_authenticated: abort(403)") == "auth_check"
|
||||
assert classify_guard_text('if name not in ALLOWED: abort(400)') == "allowlist_test"
|
||||
assert classify_guard_text("if len(buf) > 64: return") == "bounds_test"
|
||||
assert classify_guard_text("if value is None: return") == "null_test"
|
||||
assert classify_guard_text("page = int(raw)") == "type_change"
|
||||
assert classify_guard_text("total = total + 1") == "none"
|
||||
|
||||
|
||||
def test_the_seeded_families_and_the_named_gaps_are_both_exact() -> None:
|
||||
"""What the retained fact data can seed, and what it provably cannot — Req 9.2, a gap is *named*.
|
||||
|
||||
`path`, `output_encoding`, `untrusted_destination` and `prototype` have no sink in `ruleset/semantic`
|
||||
whose CWE routes to them, which is the same hole the entailment ceiling diagnosed ("closed literal sink
|
||||
table: misses per-family sinks"). Task 4.2 authors those sinks and must update this test when it does —
|
||||
the set is asserted exactly so a gap can neither appear nor close silently.
|
||||
"""
|
||||
from openultrasast.model.specs import dominance_specs, taint_specs
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
|
||||
taxonomy = load_families()
|
||||
seeded = set()
|
||||
for language in ("python", "javascript", "java", "c"):
|
||||
seeded |= {spec.family for spec in taint_specs(language=language).values()}
|
||||
seeded |= {spec.family for spec in dominance_specs(language=language).values()}
|
||||
deferred = {"unknown", "memory"} # the declined bucket and the deferred execution tier
|
||||
assert seeded == {"injection", "deserialization", "config_secrets", "access_control", "memory"}
|
||||
gaps = {family.id for family in taxonomy.families} - seeded - deferred
|
||||
assert gaps == {"path", "output_encoding", "untrusted_destination", "prototype"}, (
|
||||
"a family lost or gained a seed without this record being updated"
|
||||
)
|
||||
|
||||
|
||||
def test_the_injection_taint_spec_carries_sources_sinks_and_sanitizers() -> None:
|
||||
from openultrasast.model.specs import taint_specs
|
||||
|
||||
spec = taint_specs(language="python")["injection"]
|
||||
assert any("request.args" in source for source in spec.sources)
|
||||
assert any(sink in ("eval", "exec", "os.system") for sink in spec.sinks)
|
||||
assert spec.sanitizers, "a family with no sanitizer model can never distinguish a fixed twin"
|
||||
|
||||
|
||||
def test_the_access_control_dominance_spec_carries_operations_and_dischargers() -> None:
|
||||
from openultrasast.model.specs import dominance_specs
|
||||
|
||||
spec = dominance_specs(language="python")["access_control"]
|
||||
assert any("filter_by" in operation for operation in spec.operations)
|
||||
assert spec.dischargers, "an absence family with no discharger model can only ever report suspicion"
|
||||
|
||||
|
||||
def test_specs_import_nothing_from_the_removed_packages() -> None:
|
||||
"""Req 1.2: the vocabularies must not depend on the modules this feature deletes."""
|
||||
tree = ast.parse(SPECS.read_text())
|
||||
imported = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
imported.add(node.module)
|
||||
elif isinstance(node, ast.Import):
|
||||
imported.update(alias.name for alias in node.names)
|
||||
forbidden = ("learning", "improve", "variants", "mechanisms", "evolve")
|
||||
offending = [name for name in imported if any(token in name for token in forbidden)]
|
||||
assert not offending, f"model/specs.py imports removed code: {offending}"
|
||||
|
||||
|
||||
def test_the_specs_module_exposes_no_writer() -> None:
|
||||
"""Req 7.4, the verifier boundary: the models are read-only data no LLM or optimiser can rewrite."""
|
||||
tree = ast.parse(SPECS.read_text())
|
||||
writers = [
|
||||
node.name
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
and any(node.name.startswith(verb) for verb in ("write", "save", "dump", "store", "append", "update", "set_"))
|
||||
]
|
||||
assert not writers, f"specs must be read-only, found writers: {writers}"
|
||||
assert "open(" not in SPECS.read_text().replace("# ", ""), "specs must not open a file for writing"
|
||||
@@ -14,7 +14,7 @@ GOOD = (
|
||||
|
||||
|
||||
def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None:
|
||||
from openultrasast.learning.families import FAMILY_IDS, VERIFIER_KINDS, load_families
|
||||
from openultrasast.model.taxonomy import FAMILY_IDS, VERIFIER_KINDS, load_families
|
||||
|
||||
taxonomy = load_families()
|
||||
assert len(FAMILY_IDS) == 10 and FAMILY_IDS[-1] == "unknown"
|
||||
@@ -27,7 +27,7 @@ def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None:
|
||||
|
||||
|
||||
def test_every_mechanism_id_resolves_to_exactly_one_family() -> None:
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
from openultrasast.pairs import load_mechanisms
|
||||
|
||||
taxonomy = load_families()
|
||||
@@ -43,7 +43,7 @@ def test_every_mechanism_id_resolves_to_exactly_one_family() -> None:
|
||||
|
||||
|
||||
def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None:
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
|
||||
taxonomy = load_families()
|
||||
seen: dict[str, str] = {}
|
||||
@@ -63,7 +63,7 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None
|
||||
|
||||
|
||||
def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
|
||||
taxonomy = load_families()
|
||||
assert taxonomy.related("injection", "injection") == "same"
|
||||
@@ -109,7 +109,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
|
||||
],
|
||||
)
|
||||
def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str) -> None:
|
||||
from openultrasast.learning.families import FamiliesError, load_families
|
||||
from openultrasast.model.taxonomy import FamiliesError, load_families
|
||||
|
||||
(tmp_path / "families.toml").write_text(text)
|
||||
with pytest.raises(FamiliesError, match=needle):
|
||||
@@ -117,7 +117,7 @@ def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str
|
||||
|
||||
|
||||
def test_the_shipped_file_must_declare_the_whole_closed_set_in_order(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.families import FamiliesError, load_families
|
||||
from openultrasast.model.taxonomy import FamiliesError, load_families
|
||||
|
||||
(tmp_path / "families.toml").write_text(GOOD)
|
||||
with pytest.raises(FamiliesError, match="in that order"):
|
||||
@@ -0,0 +1,45 @@
|
||||
"""model-grounded-detection Req 3: outgrowth is visible and checkable, not noticed two specs later."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path("src/openultrasast")
|
||||
MANIFEST = Path("benchmarks/measurements/2026-09-08-module-audit.json")
|
||||
|
||||
|
||||
def test_no_module_in_the_tree_is_orphaned() -> None:
|
||||
from openultrasast.model.audit import audit
|
||||
|
||||
orphans = [row.module for row in audit(SRC) if row.classification == "orphaned"]
|
||||
assert not orphans, f"orphaned modules must be removed, not carried: {orphans}"
|
||||
|
||||
|
||||
def test_every_module_is_classified() -> None:
|
||||
from openultrasast.model.audit import audit, module_name
|
||||
|
||||
classified = {row.module for row in audit(SRC)}
|
||||
on_disk = {module_name(path, SRC) for path in SRC.rglob("*.py") if "__pycache__" not in path.parts}
|
||||
assert on_disk - classified <= {""}, f"unclassified: {sorted(on_disk - classified - {''})}"
|
||||
|
||||
|
||||
def test_the_named_subsystems_each_carry_an_explicit_decision() -> None:
|
||||
"""Req 3.3: the five older-phase subsystems get a decision, not silence."""
|
||||
from openultrasast.model.audit import audit
|
||||
|
||||
rows = {row.module: row for row in audit(SRC)}
|
||||
for name in ("fusion", "mcp", "skills", "harness_ext", "hunter_harness"):
|
||||
assert name in rows, f"{name} vanished without a recorded decision"
|
||||
assert rows[name].reason, f"{name} has no stated reason"
|
||||
|
||||
|
||||
def test_the_manifest_matches_the_tree() -> None:
|
||||
"""A later reader can check the tree against the committed manifest (Req 3.4)."""
|
||||
from openultrasast.model.audit import audit, to_dict
|
||||
|
||||
assert MANIFEST.is_file(), "the audit manifest is committed evidence, not a local artifact"
|
||||
committed = json.loads(MANIFEST.read_text())
|
||||
current = to_dict(audit(SRC))
|
||||
assert committed["counts"] == current["counts"], "the tree drifted from its committed audit"
|
||||
assert committed["orphaned"] == current["orphaned"] == []
|
||||
@@ -115,37 +115,24 @@ def test_path_records_gate_reachability_and_a_dominating_hop_discharges() -> Non
|
||||
assert covered.findings # without the hop witness the finding stands
|
||||
|
||||
|
||||
def test_findings_become_suspicion_static_findings_with_known_fix_and_labels(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.obligations.check import findings_to_static
|
||||
from openultrasast.semantic.obligations.shapes import ObligationShape, obligation_mechanisms
|
||||
def test_findings_become_suspicion_static_findings_with_labels() -> None:
|
||||
"""The finding-id contract, the suspicion rung and the tags, without the deleted mechanism store.
|
||||
|
||||
store = MechanismStore(tmp_path / "m.jsonl")
|
||||
shape = ObligationShape(
|
||||
language="python",
|
||||
operation_kind="protected_read",
|
||||
discharger_kind="identity_constraint",
|
||||
provenance="authenticated_context",
|
||||
resource_class="owned",
|
||||
mechanism="missing_auth_guard",
|
||||
)
|
||||
record = append_from_pair(store, shape, summary="s", cwe="CWE-639", pair="vampi", provenance="human", tier="seeded")
|
||||
result = _check(store_shapes=obligation_mechanisms(store.load()))
|
||||
The original also asserted `known_fix` — a hint the store supplied. The store is gone, so the hint is gone
|
||||
with it; everything the checker establishes on its own is still pinned here.
|
||||
"""
|
||||
from openultrasast.semantic.obligations.check import findings_to_static
|
||||
|
||||
result = _check()
|
||||
leaky = next(f for f in result.findings if f.operation.function == "leaky" and f.missing == "identity_constraint")
|
||||
assert leaky.known_fix == record.id
|
||||
guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint")
|
||||
statics = findings_to_static(result)
|
||||
ids = {s.finding_id for s in statics}
|
||||
guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint")
|
||||
assert f"obligation:protected_read:app.py:{leaky.operation.line}:identity_constraint" in ids # exact id contract
|
||||
assert f"obligation:protected_read:app.py:{guarded.operation.line}:identity_constraint" in ids
|
||||
static = next(s for s in statics if s.line == leaky.operation.line and "discharger:identity_constraint" in s.tags)
|
||||
assert static.evidence_level == "suspicion"
|
||||
assert {
|
||||
"obligation:protected_read",
|
||||
"discharger:identity_constraint",
|
||||
"obligation_evidence:consistency_violation",
|
||||
f"mechanism:{record.id}",
|
||||
} <= set(static.tags)
|
||||
assert {"obligation:protected_read", "discharger:identity_constraint", "obligation_evidence:consistency_violation"} <= set(static.tags)
|
||||
assert "app.py::constrained" in static.rationale
|
||||
|
||||
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
"""authorization-obligations task 1.3: obligation shapes travel through the existing exporter and store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.pairs import PairCase
|
||||
from openultrasast.semantic.mechanisms import MechanismStore
|
||||
|
||||
VULN = (
|
||||
"from flask import request\n\n\ndef get_book(book_title):\n"
|
||||
" resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n"
|
||||
" book = Book.query.filter_by(book_title=book_title).first()\n return book\n"
|
||||
)
|
||||
FIXED = (
|
||||
"from flask import request\n\n\ndef get_book(book_title):\n"
|
||||
" resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n"
|
||||
" user = User.query.filter_by(username=resp['sub']).first()\n"
|
||||
" book = Book.query.filter_by(user=user, book_title=book_title).first()\n return book\n"
|
||||
)
|
||||
FLOW_VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n"
|
||||
FLOW_FIXED = (
|
||||
"from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n"
|
||||
+ " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n"
|
||||
)
|
||||
|
||||
|
||||
def _case(
|
||||
tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, mechanism: str, tier: str = "seeded", sink: str | None = None
|
||||
) -> PairCase:
|
||||
(tmp_path / f"{name}-v.py").write_text(vuln)
|
||||
(tmp_path / f"{name}-f.py").write_text(fixed)
|
||||
return PairCase(
|
||||
name=name,
|
||||
slice="vibe-py",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=tmp_path / f"{name}-v.py",
|
||||
fixed_file=tmp_path / f"{name}-f.py",
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function=function, mechanism=mechanism, sink=sink
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
provenance="human",
|
||||
review_tier=tier,
|
||||
reviewer="t" if tier == "reviewed" else "",
|
||||
)
|
||||
|
||||
|
||||
def test_exporter_writes_obligation_rows_deduped_with_provenance_and_skips_untrusted(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.obligations.shapes import obligation_mechanisms
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
cases = [
|
||||
_case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard"),
|
||||
_case(
|
||||
tmp_path,
|
||||
"b",
|
||||
VULN.replace("book", "item"),
|
||||
FIXED.replace("book", "item"),
|
||||
function="get_item",
|
||||
mechanism="missing_auth_guard",
|
||||
tier="reviewed",
|
||||
),
|
||||
_case(tmp_path, "advisory", VULN, FIXED, function="get_book", mechanism="missing_auth_guard", tier="advisory"),
|
||||
_case(tmp_path, "flow", FLOW_VULN, FLOW_FIXED, function="run", mechanism="source_reaches_sink", sink="os.system"),
|
||||
]
|
||||
report = export_mechanisms(cases, store)
|
||||
rows = store.load()
|
||||
obligations = obligation_mechanisms(rows)
|
||||
assert len(obligations) == 1 and obligations[0].pairs == ("a", "b") and obligations[0].origin == "corpus"
|
||||
assert (
|
||||
obligations[0].shape is not None
|
||||
and obligations[0].shape["family"] == "obligation"
|
||||
and obligations[0].guard == "identity_constraint"
|
||||
)
|
||||
sinks = [r for r in rows if r.shape and r.shape.get("family", "sink") != "obligation"]
|
||||
assert len(sinks) == 1 and sinks[0].shape is not None and sinks[0].shape["sink_name"] == "system" # flow rows unaffected
|
||||
assert report.seeded == 3 and dict(report.skipped)["advisory"] == "tier:advisory"
|
||||
text = (tmp_path / "mechanisms.jsonl").read_text()
|
||||
assert "book_title" not in text and "resp" not in text and "app.py" not in text
|
||||
|
||||
|
||||
def test_absence_pair_without_a_lesson_is_skipped_with_the_derivation_reason(tmp_path: Path) -> None:
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
store = MechanismStore(tmp_path / "m.jsonl")
|
||||
report = export_mechanisms([_case(tmp_path, "same", VULN, VULN, function="get_book", mechanism="missing_auth_guard")], store)
|
||||
assert store.load() == () and dict(report.skipped)["same"].startswith("no_discharger_added:identity_constraint")
|
||||
|
||||
|
||||
def test_sink_search_ignores_obligation_rows(tmp_path: Path) -> None:
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.semantic.facts import load_facts
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
from openultrasast.semantic.variant_search import search_tree
|
||||
|
||||
store = MechanismStore(tmp_path / "m.jsonl")
|
||||
export_mechanisms([_case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard")], store)
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "views.py").write_text(VULN)
|
||||
_, targets = preprocess_repository(repo)
|
||||
result = search_tree(repo, targets, store, load_facts(), max_mechanisms=500)
|
||||
assert result.mechanisms_searched == 0 and result.hits == () and result.degradations == ()
|
||||
|
||||
|
||||
def test_the_train_split_of_vibe_py_teaches_no_obligation_shape_today(tmp_path: Path) -> None:
|
||||
"""learning-harness Req 5.1 turned this number honest, and the honest number is zero.
|
||||
|
||||
Every vibe-py absence pair that yields an obligation lesson sits on the holdout side, so once only
|
||||
train-split pairs may teach, the corpus teaches no obligation shape at all. The earlier result
|
||||
("at least one obligation shape from vibe-py") was true only because holdout pairs were teaching.
|
||||
The derivation itself stays covered by the fixture tests above; this pins the corpus gap, which the
|
||||
absence re-harvest and a maintainer pass over the splits are what close.
|
||||
"""
|
||||
from openultrasast.pairs import load_pair_catalog, select_slice, select_vendored
|
||||
from openultrasast.semantic.obligations.shapes import obligation_mechanisms
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
store = MechanismStore(tmp_path / "m.jsonl")
|
||||
cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py"))
|
||||
report = export_mechanisms(cases, store)
|
||||
assert obligation_mechanisms(store.load()) == ()
|
||||
refused = {pair: reason for pair, reason in report.skipped}
|
||||
assert refused["vampi-users-update-password"] == "split:holdout"
|
||||
assert refused["threatbyte-api-v1-get"] == "split:holdout"
|
||||
# Its own reason, not its split: this row is a byte-identical twin that happens to sit on the holdout side,
|
||||
# and reporting `split:holdout` would hide a corpus fact behind a split fact (Req 5.2).
|
||||
assert refused["vampi-books-get-by-title"] == "identical_twin"
|
||||
flow_rows = [row for row in store.load() if row.shape is not None and row.shape.get("family", "sink") == "sink"]
|
||||
assert report.records == len(flow_rows) and len(flow_rows) >= 1 # the flow family still teaches from the train split
|
||||
|
||||
|
||||
def test_absence_row_without_an_obligation_lesson_still_teaches_its_sink_shape(tmp_path: Path) -> None:
|
||||
"""The fallback that keeps the flow store unchanged: a `missing_auth_guard` row whose twin adds no discharger but whose
|
||||
labeled sink call carries a source still seeds a sink shape (the threatbyte pairs do exactly this)."""
|
||||
from openultrasast.semantic.obligations.shapes import obligation_mechanisms
|
||||
from openultrasast.semantic.seed import export_mechanisms
|
||||
|
||||
vuln = (
|
||||
"import sqlite3\n\n\ndef get(user_id):\n cur = sqlite3.connect('x').cursor()\n"
|
||||
" cur.execute('SELECT * FROM users WHERE id = ' + user_id)\n return cur.fetchone()\n"
|
||||
)
|
||||
fixed = (
|
||||
"import sqlite3\n\n\ndef login(username):\n cur = sqlite3.connect('x').cursor()\n"
|
||||
" cur.execute('SELECT * FROM users WHERE name = ?', (username,))\n return cur.fetchone()\n"
|
||||
)
|
||||
store = MechanismStore(tmp_path / "m.jsonl")
|
||||
report = export_mechanisms([_case(tmp_path, "tb", vuln, fixed, function="get", mechanism="missing_auth_guard", sink="execute")], store)
|
||||
rows = store.load()
|
||||
assert report.seeded == 1 and obligation_mechanisms(rows) == ()
|
||||
assert len(rows) == 1 and rows[0].shape is not None and rows[0].shape.get("family", "sink") == "sink"
|
||||
assert rows[0].shape["sink_name"] == "execute"
|
||||
@@ -370,55 +370,6 @@ def test_known_limit_pairs_are_evaluated_but_excluded_from_achievable(tmp_path:
|
||||
assert set(result.per_mechanism) == {"cross_artifact", "source_reaches_sink"}
|
||||
assert result.loss["sast"]["known_limit"] == 1
|
||||
assert "overlay" in result.scorers["sast"] and "inventory" in result.scorers["sast"]
|
||||
assert result.degradations and result.degradations[0]["reason"] == "hunter_model_unavailable"
|
||||
|
||||
|
||||
def test_hunter_scorer_credits_a_family_tag_and_never_the_words(tmp_path: Path) -> None:
|
||||
"""learning-harness Req 3.1/3.6 replaced this path's rule: the family tag decides, the rationale never does."""
|
||||
from openultrasast.findings import StaticFinding
|
||||
|
||||
src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n"
|
||||
row = ExpectedFinding(
|
||||
cwe="CWE-95",
|
||||
vulnerability_class="code injection",
|
||||
path="app.py",
|
||||
evidence="",
|
||||
sink="eval",
|
||||
function="f",
|
||||
mechanism="source_reaches_sink",
|
||||
)
|
||||
|
||||
def scripted(tags: list[str]): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if "eval(" not in (root / "app.py").read_text():
|
||||
return []
|
||||
return [
|
||||
StaticFinding(
|
||||
finding_id="tool-hunter:app.py:4",
|
||||
path="app.py",
|
||||
title="eval of request input",
|
||||
severity="high",
|
||||
confidence="medium",
|
||||
evidence_level="suspicion",
|
||||
rationale="CWE-95 eval executes attacker input",
|
||||
line=4,
|
||||
function_name=None,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=tags,
|
||||
ranking_priority=1.0,
|
||||
)
|
||||
]
|
||||
|
||||
return scan
|
||||
|
||||
case = _case(tmp_path, "hunt", src, "def f():\n return 1\n", row)
|
||||
tagged = evaluate_catalog([case], hunter=scripted(["family:injection"])).scorers["sast"]["hunter"]
|
||||
assert tagged.pair_correct == 1 and tagged.detected_vuln == 1 and tagged.silent_fix == 1
|
||||
# the same finding, the same words, no family tag: not a detection, because the rationale is not evidence
|
||||
untagged = evaluate_catalog([case], hunter=scripted([])).scorers["sast"]["hunter"]
|
||||
assert untagged.detected_vuln == 0 and untagged.silent_fix == 1
|
||||
|
||||
|
||||
def test_juliet_strcpy_pair_bodies_differ_and_java_hash_is_known_limit() -> None:
|
||||
|
||||
@@ -17,8 +17,8 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learning.families import load_families
|
||||
from openultrasast.mapping import analyze_entry_points
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.semantic.extra import has_semantic_extra
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""MAP overlay, prove filter, reports, pair scoring, mechanism memory."""
|
||||
"""MAP overlay, prove filter, reports, pair scoring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -15,7 +15,7 @@ from openultrasast.gate import LANGUAGE_MANIFESTS
|
||||
from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.rank import rank_targets
|
||||
from openultrasast.semantic import MechanismStore, OverlayRecord, append_mechanism, filter_promoted_hotspots, order_promotions
|
||||
from openultrasast.semantic import OverlayRecord, filter_promoted_hotspots
|
||||
from openultrasast.semantic.prove_filter import promoted_findings
|
||||
|
||||
|
||||
@@ -265,79 +265,3 @@ def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None:
|
||||
assert "coverage" in text
|
||||
|
||||
|
||||
def test_mechanism_order_boosts_paraphrased_match(tmp_path: Path) -> None:
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
append_mechanism(
|
||||
store,
|
||||
summary="request parameter reaches eval",
|
||||
cwe="CWE-95",
|
||||
language="python",
|
||||
tags=["syscall_entry"],
|
||||
what_made_it_exploitable="dynamic execution of attacker input",
|
||||
)
|
||||
eval_id = "python-unsafe-eval:app.py:1"
|
||||
hash_id = "python-weak-hash:hash.py:1"
|
||||
hotspots = (_hotspot(hash_id, score=5.0, path="hash.py"), _hotspot(eval_id, score=5.0, path="app.py"))
|
||||
records = (
|
||||
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",), language="python"),
|
||||
_record(eval_id, "promote", sources=("request",), cwe="CWE-95", sinks=("eval",), language="python"),
|
||||
)
|
||||
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
||||
|
||||
class _Client:
|
||||
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
||||
del model, timeout_seconds
|
||||
vectors: list[list[float]] = []
|
||||
for text in inputs:
|
||||
if "eval" in text.lower() or "CWE-95" in text:
|
||||
vectors.append([1.0, 0.0])
|
||||
else:
|
||||
vectors.append([0.0, 1.0])
|
||||
return vectors
|
||||
|
||||
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=_Client(), model="test-embed")
|
||||
assert degradation is None
|
||||
assert ordered[0].inventory_finding_ids == (eval_id,)
|
||||
assert {item.inventory_finding_ids[0] for item in ordered} == {eval_id, hash_id}
|
||||
|
||||
|
||||
def test_mechanism_degrades_to_heuristic_without_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
append_mechanism(
|
||||
store,
|
||||
summary="request parameter reaches eval",
|
||||
cwe="CWE-95",
|
||||
language="python",
|
||||
tags=["syscall_entry"],
|
||||
what_made_it_exploitable="dynamic execution of attacker input",
|
||||
)
|
||||
eval_id = "python-unsafe-eval:app.py:1"
|
||||
hash_id = "python-weak-hash:hash.py:1"
|
||||
hotspots = (_hotspot(hash_id, score=6.0, path="hash.py"), _hotspot(eval_id, score=3.0, path="app.py"))
|
||||
records = (
|
||||
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",)),
|
||||
_record(eval_id, "promote", sources=("request",)),
|
||||
)
|
||||
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
||||
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=None, model=None)
|
||||
assert degradation == "embeddings_unavailable"
|
||||
assert [item.inventory_finding_ids[0] for item in ordered] == [hash_id, eval_id]
|
||||
|
||||
|
||||
def test_mechanisms_are_not_written_from_unadjudicated(tmp_path: Path) -> None:
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
record = OverlayRecord(
|
||||
proposal_id="python-unsafe-eval:app.py:1",
|
||||
path="app.py",
|
||||
line=1,
|
||||
disposition="unadjudicated",
|
||||
reason="parse_failed",
|
||||
cwe="CWE-95",
|
||||
sources=(),
|
||||
sinks=(),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
)
|
||||
assert record.disposition != "promote"
|
||||
assert store.load() == ()
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""model-grounded-detection Req 1.3: nothing survives that imports what this feature removed.
|
||||
|
||||
The noise architecture and the mechanism/evolve loop are deleted, not deprecated. This walks the AST of every
|
||||
surviving module and fails on any import that names one of them — the check that turns a five-thousand-line
|
||||
deletion from crash-driven discovery into one assertion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path("src/openultrasast")
|
||||
|
||||
REMOVED_MODULES = frozenset(
|
||||
{
|
||||
# the noise architecture (the whole learning/ package; taxonomy, candidates and endpoint re-homed to model/)
|
||||
"learning",
|
||||
# the mechanism/evolve optimisation loop
|
||||
"semantic.mechanisms",
|
||||
"semantic.variants",
|
||||
"semantic.variant_search",
|
||||
"semantic.prove_budget",
|
||||
"semantic.loo",
|
||||
"semantic.seed",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _module_names(tree: ast.AST) -> set[str]:
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
names.add(node.module)
|
||||
elif isinstance(node, ast.Import):
|
||||
names.update(alias.name for alias in node.names)
|
||||
return names
|
||||
|
||||
|
||||
def _names_a_removed_module(imported: str) -> bool:
|
||||
trimmed = imported.lstrip(".")
|
||||
return any(trimmed == removed or trimmed.startswith(f"{removed}.") for removed in REMOVED_MODULES)
|
||||
|
||||
|
||||
def test_no_surviving_module_imports_a_removed_one() -> None:
|
||||
offenders: dict[str, list[str]] = {}
|
||||
for path in sorted(SRC.rglob("*.py")):
|
||||
named = sorted(name for name in _module_names(ast.parse(path.read_text())) if _names_a_removed_module(name))
|
||||
if named:
|
||||
offenders[str(path)] = named
|
||||
assert not offenders, f"dangling imports of removed modules: {offenders}"
|
||||
|
||||
|
||||
def test_the_removed_modules_are_gone_from_the_tree() -> None:
|
||||
assert not (SRC / "learning").exists(), "the learning package is removed; taxonomy/candidates/endpoint live under model/"
|
||||
for relative in ("semantic/mechanisms.py", "semantic/variants.py", "semantic/variant_search.py",
|
||||
"semantic/prove_budget.py", "semantic/loo.py", "semantic/seed.py"):
|
||||
assert not (SRC / relative).exists(), f"{relative} still exists"
|
||||
|
||||
|
||||
def test_the_mechanism_lever_is_gone_from_the_retained_improve_loop() -> None:
|
||||
"""`improve/` survives with its lever removed, not deleted.
|
||||
|
||||
The design's Removed list named `improve/evolve.py` whole, but the file is a mix: the mechanism lever
|
||||
(deleted here) and the deterministic rule-status loop that `ousast improve` still runs from benchmark
|
||||
evidence, which has no LLM in it and is not what this feature supersedes. Deleting the file would have
|
||||
taken a retained capability with it, so the lever is stripped and the loop stays — asserted by symbol so
|
||||
the lever cannot creep back.
|
||||
"""
|
||||
evolve = (SRC / "improve" / "evolve.py").read_text()
|
||||
validator = (SRC / "improve" / "validator.py").read_text()
|
||||
for symbol in ("MechanismEdit", "MechanismStore", "apply_mechanism_edits", "evaluate_mechanism_profiles",
|
||||
"propose_mechanism_edits", "validate_mechanism", "mechanism_store", "mechanism_candidates"):
|
||||
assert symbol not in evolve, f"{symbol} survives in improve/evolve.py"
|
||||
assert symbol not in validator, f"{symbol} survives in improve/validator.py"
|
||||
assert "def run_improvement(" in evolve and "def propose_status_edits(" in evolve, "the rule-status loop must survive"
|
||||
|
||||
|
||||
def test_the_re_homed_modules_are_where_the_model_layer_expects_them() -> None:
|
||||
for relative in ("model/taxonomy.py", "model/candidates.py", "model/endpoint.py", "model/specs.py"):
|
||||
assert (SRC / relative).is_file(), f"{relative} is missing"
|
||||
@@ -1,130 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 2.3: variant findings in a standard scan, sandbox-eligible, counted in the manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
|
||||
VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n"
|
||||
UNSEEN_SINK = (
|
||||
"from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n" # a project helper: no inventory rule, no fact sink
|
||||
)
|
||||
|
||||
|
||||
def _seed(repo: Path) -> str:
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
shape = Shape(
|
||||
language="python",
|
||||
sink_name="run_shell",
|
||||
arity=1,
|
||||
source_positions=(0,),
|
||||
source_kinds=("parameter",),
|
||||
guard="allowlist_test",
|
||||
mechanism="source_reaches_sink",
|
||||
)
|
||||
store = MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl")
|
||||
return append_from_pair(
|
||||
store, shape, summary="parameter into run_shell; fix adds allowlist", cwe="CWE-78", pair="p1", provenance="human", tier="seeded"
|
||||
).id
|
||||
|
||||
|
||||
def _latest_run(repo: Path) -> Path:
|
||||
runs = sorted((repo / ".runs").iterdir())
|
||||
return runs[-1]
|
||||
|
||||
|
||||
def test_standard_scan_emits_variant_findings_and_manifest_counters(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(UNSEEN_SINK) # only the learned shape can find a project-specific sink
|
||||
mechanism_id = _seed(repo)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
variant = [f for f in findings if f["finding_id"].startswith("variant:")]
|
||||
assert len(variant) == 1 and variant[0]["finding_id"] == f"variant:{mechanism_id}:app.py:5"
|
||||
assert variant[0]["evidence_level"] == "suspicion" and f"mechanism:{mechanism_id}" in variant[0]["tags"]
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert manifest["variants"] == {"mechanisms_searched": 1, "files_searched": 1, "findings": 1, "merged_into_overlay": 0}
|
||||
|
||||
|
||||
def test_variant_hit_merges_into_an_overlay_flow_at_the_same_call_site(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(VARIANT) # os.system is both an inventory rule and a fact sink: the overlay records the flow
|
||||
shape = Shape(
|
||||
language="python",
|
||||
sink_name="system",
|
||||
arity=1,
|
||||
source_positions=(0,),
|
||||
source_kinds=("fact_source",),
|
||||
guard="none",
|
||||
mechanism="source_reaches_sink",
|
||||
)
|
||||
record = append_from_pair(
|
||||
MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"),
|
||||
shape,
|
||||
summary="s",
|
||||
cwe="CWE-78",
|
||||
pair="p",
|
||||
provenance="human",
|
||||
tier="seeded",
|
||||
)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
|
||||
assert any(item["mechanism_id"] == record.id and item["line"] == 7 for item in overlay)
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
assert not any(f["finding_id"].startswith("variant:") for f in findings) # merged, not duplicated
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert manifest["variants"]["merged_into_overlay"] == 1 and manifest["variants"]["findings"] == 0
|
||||
# Req 6.1 end to end: the merged inventory finding names the mechanism in both reports and keeps its own evidence level
|
||||
report = (run_dir / "report.md").read_text()
|
||||
assert f"- Mechanism: `{record.id}`" in report and "- Learned from pairs: `p`" in report
|
||||
sarif = json.loads((run_dir / "report.sarif").read_text())
|
||||
props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"].get("mechanism_id") == record.id]
|
||||
assert props and props[0]["mechanism_guard"] == "none" and props[0]["evidence_level"] != "suspicion"
|
||||
|
||||
|
||||
def test_quick_scan_never_searches_variants(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(UNSEEN_SINK)
|
||||
_seed(repo)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert "variants" not in manifest
|
||||
assert not any(f["finding_id"].startswith("variant:") for f in json.loads((run_dir / "findings.json").read_text())["findings"])
|
||||
|
||||
|
||||
def test_deep_scan_offers_variant_findings_to_the_sandbox_after_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast import cli
|
||||
from openultrasast.sandbox import FakeSandboxRunner, SandboxResult
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(UNSEEN_SINK)
|
||||
mechanism_id = _seed(repo)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
|
||||
fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False))
|
||||
monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake)
|
||||
assert main(["scan", str(repo), "--mode", "deep"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
|
||||
offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]}
|
||||
assert f"variant:{mechanism_id}:app.py:5" in offered
|
||||
@@ -1,124 +0,0 @@
|
||||
"""corpus-seeded-mechanisms task 2.2: structural variant search over a scanned tree, merged with overlay flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.semantic.facts import load_facts
|
||||
from openultrasast.semantic.ir import parse_file
|
||||
from openultrasast.semantic.overlay import OverlayRecord
|
||||
|
||||
PY_FACTS = load_facts()
|
||||
|
||||
|
||||
def _shape(**overrides: object):
|
||||
from openultrasast.semantic.variants import Shape
|
||||
|
||||
base: dict[str, object] = {
|
||||
"language": "python",
|
||||
"sink_name": "system",
|
||||
"arity": 1,
|
||||
"source_positions": (0,),
|
||||
"source_kinds": ("fact_source",),
|
||||
"guard": "allowlist_test",
|
||||
"mechanism": "source_reaches_sink",
|
||||
}
|
||||
base.update(overrides)
|
||||
return Shape(**base) # type: ignore[arg-type]
|
||||
|
||||
|
||||
VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n"
|
||||
PARAM_VARIANT = "import os\n\n\ndef handler(q):\n os.system(q)\n"
|
||||
CONSTANT = "import os\n\n\ndef handler():\n os.system('ls -la')\n"
|
||||
ARITY = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q, 1)\n"
|
||||
|
||||
|
||||
def test_match_shapes_finds_structural_variants_and_ignores_constants() -> None:
|
||||
from openultrasast.semantic.variants import match_shapes
|
||||
|
||||
shape = _shape()
|
||||
hits = match_shapes(parse_file("a.py", VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"})
|
||||
assert [(h.path, h.line, h.mechanism_id, h.sink_name, h.source_kind) for h in hits] == [
|
||||
("a.py", 7, "corpus:abc", "system", "fact_source")
|
||||
]
|
||||
param_hits = match_shapes(parse_file("b.py", PARAM_VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"})
|
||||
assert [(h.line, h.source_kind) for h in param_hits] == [(5, "parameter")] # any source kind at the shape's positions matches
|
||||
assert match_shapes(parse_file("c.py", CONSTANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == []
|
||||
assert match_shapes(parse_file("d.py", ARITY, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == []
|
||||
js_shape = _shape(language="javascript", sink_name="exec")
|
||||
assert match_shapes(parse_file("a.py", VARIANT, "python"), [js_shape], PY_FACTS, mechanism_ids={js_shape.key(): "x"}) == []
|
||||
|
||||
|
||||
def test_hits_become_suspicion_findings_or_merge_into_an_overlay_flow() -> None:
|
||||
from openultrasast.semantic.variant_search import hits_to_findings
|
||||
from openultrasast.semantic.variants import VariantHit
|
||||
|
||||
hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="fact_source")
|
||||
other = VariantHit(path="z.py", line=3, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter")
|
||||
coverage = OverlayRecord(
|
||||
proposal_id="overlay-coverage:a.py:7:os.system",
|
||||
path="a.py",
|
||||
line=7,
|
||||
disposition="coverage",
|
||||
reason="uninventoried sink os.system reached from request",
|
||||
cwe="CWE-78",
|
||||
sources=("request",),
|
||||
sinks=("os.system",),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
origin="overlay",
|
||||
engine="python-ast",
|
||||
language="python",
|
||||
)
|
||||
summaries = {"corpus:abc": ("request into os.system; fix adds allowlist_test", ("p1", "p2"), "CWE-78")}
|
||||
findings, records = hits_to_findings([hit, other], [coverage], summaries)
|
||||
assert len(records) == 1 and records[0].mechanism_id == "corpus:abc" and records[0].disposition == "coverage"
|
||||
assert [f.finding_id for f in findings] == ["variant:corpus:abc:z.py:3"] # the a.py hit merged, no duplicate
|
||||
(finding,) = findings
|
||||
assert finding.evidence_level == "suspicion" and "variant" in finding.tags and "mechanism:corpus:abc" in finding.tags
|
||||
assert "p1" in finding.rationale and finding.line == 3 and finding.path == "z.py"
|
||||
|
||||
|
||||
def test_search_tree_uses_the_store_and_respects_the_mechanism_cap(tmp_path: Path) -> None:
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
||||
from openultrasast.semantic.variant_search import search_tree
|
||||
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "views.py").write_text(VARIANT)
|
||||
(root / "safe.py").write_text(CONSTANT)
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
record = append_from_pair(store, _shape(), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded")
|
||||
_, targets = preprocess_repository(root)
|
||||
result = search_tree(root, targets, store, PY_FACTS, max_mechanisms=500)
|
||||
assert [(h.path, h.line, h.mechanism_id) for h in result.hits] == [("views.py", 7, record.id)]
|
||||
assert result.mechanisms_searched == 1 and result.files_searched == 2
|
||||
capped = search_tree(root, targets, store, PY_FACTS, max_mechanisms=0)
|
||||
assert capped.hits == () and capped.mechanisms_searched == 0
|
||||
|
||||
|
||||
def test_hits_never_merge_into_demoted_or_flowless_records() -> None:
|
||||
from openultrasast.semantic.variant_search import hits_to_findings
|
||||
from openultrasast.semantic.variants import VariantHit
|
||||
|
||||
hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter")
|
||||
demoted = OverlayRecord(
|
||||
proposal_id="python-os-command:a.py:7",
|
||||
path="a.py",
|
||||
line=7,
|
||||
disposition="demote",
|
||||
reason="constant argument",
|
||||
cwe="CWE-78",
|
||||
sources=(),
|
||||
sinks=("os.system",),
|
||||
sanitizers=(),
|
||||
evidence_level="suspicion",
|
||||
dominating_fact="constant",
|
||||
origin="inventory",
|
||||
engine="python-ast",
|
||||
language="python",
|
||||
)
|
||||
findings, records = hits_to_findings([hit], [demoted], {})
|
||||
assert records[0].mechanism_id is None and records[0].disposition == "demote" # Req 3.6: never demote, never re-label a demotion
|
||||
assert [f.finding_id for f in findings] == ["variant:corpus:abc:a.py:7"]
|
||||
@@ -1,76 +0,0 @@
|
||||
"""corpus-seeded-mechanisms: structural shapes derived from pairs and matched on trees (offline, stdlib ast)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openultrasast.semantic.facts import load_facts
|
||||
from openultrasast.semantic.ir import parse_file
|
||||
|
||||
VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n"
|
||||
FIXED = (
|
||||
"from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n"
|
||||
" if cmd not in ALLOWED:\n return None\n os.system(cmd)\n"
|
||||
)
|
||||
|
||||
|
||||
def _derive(vuln: str = VULN, fixed: str = FIXED, **overrides: object):
|
||||
from openultrasast.semantic.variants import derive_shape
|
||||
|
||||
kwargs: dict[str, object] = {"function": "run", "sink": "os.system", "line": None, "mechanism": "source_reaches_sink"}
|
||||
kwargs.update(overrides)
|
||||
return derive_shape(
|
||||
parse_file("app.py", vuln, "python"),
|
||||
parse_file("app.py", fixed, "python"),
|
||||
vuln_text=vuln,
|
||||
fixed_text=fixed,
|
||||
facts=load_facts(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_derive_shape_from_a_request_to_os_system_pair_with_an_allowlist_fix() -> None:
|
||||
from openultrasast.semantic.variants import GUARD_KINDS, SOURCE_KINDS
|
||||
|
||||
shape = _derive()
|
||||
assert shape is not None
|
||||
assert shape.language == "python" and shape.sink_name == "system" and shape.arity == 1
|
||||
assert shape.source_positions == (0,) and shape.source_kinds == ("fact_source",)
|
||||
assert shape.guard == "allowlist_test" and shape.guard in GUARD_KINDS and shape.mechanism == "source_reaches_sink"
|
||||
assert set(shape.source_kinds) <= set(SOURCE_KINDS)
|
||||
|
||||
|
||||
def test_shape_key_is_stable_and_carries_no_path_line_or_literal() -> None:
|
||||
a, b = _derive(), _derive()
|
||||
assert a is not None and b is not None and a.key() == b.key()
|
||||
key = a.key()
|
||||
for forbidden in ("app.py", "cmd", "'ls'", "request.args", ":6", ":7"):
|
||||
assert forbidden not in key
|
||||
# the same code in another file with different variable names is the same shape
|
||||
other_vuln = VULN.replace("cmd", "command")
|
||||
other = _derive(other_vuln, FIXED.replace("cmd", "command"))
|
||||
assert other is not None and other.key() == key
|
||||
|
||||
|
||||
def test_parameter_and_container_read_sources_are_classified() -> None:
|
||||
vuln = "import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n os.system(cmd + target)\n"
|
||||
fixed = (
|
||||
"import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n if len(target) > 32:\n return None\n"
|
||||
" os.system(cmd + target)\n"
|
||||
)
|
||||
shape = _derive(vuln, fixed)
|
||||
assert shape is not None
|
||||
assert shape.source_positions == (0,)
|
||||
assert shape.source_kinds in {("parameter",), ("container_read",)} # one argument carrying both; the kind is the first found
|
||||
assert shape.guard == "bounds_test"
|
||||
|
||||
|
||||
def test_unlabeled_or_missing_sink_returns_none_and_guard_defaults_to_none() -> None:
|
||||
assert _derive(sink="subprocess.run") is None # the label names a sink the function never calls
|
||||
assert _derive(function="other") is None # no such function
|
||||
same = _derive(VULN, VULN)
|
||||
assert same is not None and same.guard == "none" # no statement differs on the fixed side
|
||||
constant = "import os\n\n\ndef run():\n os.system('ls')\n"
|
||||
assert _derive(constant, constant) is None # a constant argument carries no source: nothing to learn
|
||||
|
||||
|
||||
def test_parse_failed_side_yields_no_shape() -> None:
|
||||
assert _derive("def run(:\n", FIXED) is None
|
||||
Reference in New Issue
Block a user