mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
feat(model-grounded-detection): group 1, subtract the noise architecture
99 source modules -> 81. 832 tests -> 646. The whole learning/ package (16 modules, ~4,100 lines) and six semantic/ modules removed, with the three gates byte-identical to 2026-09-06-gate-baseline.txt at every step. The sequence is what made it safe: vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion third (1.4). Every import break during the deletion was therefore a detachment miss in the importer and was fixed there -- no module was restored to satisfy one. Both misses were the same shape and were fixed as a class: operations.py and shapes.py each imported trailing_name from the deleted variants.py, so it moved to semantic/ir.py, its natural home, and both importers changed in one edit. Ported before deleting: the closed guard vocabulary (verbatim, nothing else holds it) into model/specs.py, which derives TaintSpec and DominanceSpec from the retained ruleset facts rather than copying them. Re-homed: families -> model/taxonomy, candidates -> model/candidates, endpoint -> model/endpoint, plus Region, labeled_family and labeled_spans. labeled_family no longer falls back to the LLM classifier -- an unlabeled pair is honestly unknown. Four deviations, each recorded in the spec notes with its reason. improve/ evolve.py and validator.py were stripped, not deleted: each mixes the mechanism lever (gone) with the deterministic rule-status loop `ousast improve` still runs, which has no LLM in it; a test now asserts the lever's absence by symbol so it cannot creep back. The learning CLI group became `model candidates`, because deleting it outright would have orphaned the retained enumerator by removing its only entry point. thinking moved from [learning] to [models]. A gap named rather than papered over: no retained sink fact routes by CWE to path, output_encoding, untrusted_destination or prototype -- the same hole the entailment ceiling diagnosed. The test asserts the seeded set and the gap set exactly, so task 4.2 must update it when it authors those sinks. The audit found exactly one genuine orphan: vectorstore.py, a bakeoff harness with no production caller and no CLI, alive only through its own test. The five named subsystems each carry an explicit decision. Manifest committed, with a test asserting the tree still matches it. Group 2 (the Joern go/no-go) has not started. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
fc4dab97fb
commit
9ed0e45c5f
@@ -26,9 +26,8 @@ from ..preprocess import preprocess_repository
|
||||
from ..rank import rank_targets
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger
|
||||
from ..scoring import build_score_artifact
|
||||
from ..semantic.facts import SemanticFacts
|
||||
from .journal import append_round, load_journal, next_round_index, reverted_edit_keys
|
||||
from .validator import EvolveValidator, MechanismEdit, RuleStatusEdit, StrictValidationError, apply_mechanism_edits, edits_to_ledger
|
||||
from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -46,9 +45,6 @@ class RoundOutcome:
|
||||
matched_before: int = 0
|
||||
matched_after: int = 0
|
||||
profile_regressions: list[str] = field(default_factory=list)
|
||||
mechanism_edits: list[MechanismEdit] = field(default_factory=list)
|
||||
mechanisms_before: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
mechanisms_after: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
profiles_under_minimum: list[str] = field(default_factory=list)
|
||||
per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
@@ -89,121 +85,6 @@ def evaluate_profiles(
|
||||
}
|
||||
|
||||
|
||||
def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]:
|
||||
"""Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers).
|
||||
|
||||
Split-blind for the same reason as `evaluate_profiles`: the store is the thing the teacher rule governs, and this
|
||||
function only reads it (learning-harness Req 5.1)."""
|
||||
from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored
|
||||
from ..semantic.loo import score_pair_with_store
|
||||
from ..semantic.mechanisms import MechanismStore
|
||||
|
||||
cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS))
|
||||
store = MechanismStore(store_path)
|
||||
by_profile: dict[str, list[tuple[bool, bool]]] = {}
|
||||
for case in cases:
|
||||
outcome = score_pair_with_store(case, store, _facts())
|
||||
by_profile.setdefault(case.provenance, []).append((outcome.detected, outcome.silent))
|
||||
result: dict[str, dict[str, float]] = {}
|
||||
for profile, items in sorted(by_profile.items()):
|
||||
pairs = len(items)
|
||||
correct = sum(1 for detected, silent in items if detected and silent)
|
||||
recall = sum(1 for detected, _ in items if detected) / pairs
|
||||
silence = sum(1 for _, silent in items if silent) / pairs
|
||||
result[profile] = {
|
||||
"pairs": float(pairs),
|
||||
"pair_correct": float(correct),
|
||||
"pair_pass_rate": correct / pairs,
|
||||
"youden": recall + silence - 1.0,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def propose_mechanism_edits(
|
||||
pair_cases: Sequence[object],
|
||||
candidates: object,
|
||||
store_path: Path,
|
||||
degradations: list[dict[str, object]] | None = None,
|
||||
) -> list[MechanismEdit]:
|
||||
"""Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker."""
|
||||
from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored
|
||||
from ..semantic.loo import score_pair_with_store
|
||||
from ..semantic.mechanisms import MechanismStore, corpus_mechanisms
|
||||
|
||||
cases_all = [case for case in pair_cases if isinstance(case, PairCase)]
|
||||
cases = list(select_tier(select_vendored(cases_all), GATING_TIERS))
|
||||
if not cases:
|
||||
return []
|
||||
facts = _facts()
|
||||
scan = MechanismStore(store_path)
|
||||
admitted = {record.id: record for record in corpus_mechanisms(scan.load())}
|
||||
current = {case.name: score_pair_with_store(case, scan, facts) for case in cases}
|
||||
edits: list[MechanismEdit] = []
|
||||
leakers = sorted({mechanism_id for outcome in current.values() for mechanism_id in outcome.leaked_by if mechanism_id in admitted})
|
||||
for mechanism_id in leakers:
|
||||
leaked = sorted(name for name, outcome in current.items() if mechanism_id in outcome.leaked_by)
|
||||
edits.append(
|
||||
MechanismEdit(
|
||||
action="retract", mechanism_id=mechanism_id, source="loo", rationale=f"leaks on the fixed side of {', '.join(leaked)}"
|
||||
)
|
||||
)
|
||||
missed = [case for case in cases if not current[case.name].detected]
|
||||
from ..learning.split import refuse_if_holdout
|
||||
|
||||
for record in corpus_mechanisms(candidates.load()): # type: ignore[attr-defined]
|
||||
if record.id in admitted or not missed:
|
||||
continue
|
||||
refusal = refuse_if_holdout(record.pairs, cases_all)
|
||||
if refusal is not None:
|
||||
# learning-harness Req 5.2: a shape a holdout pair taught can never recover that pair, and the run
|
||||
# says which pairs it was rather than showing a shorter list with no reason.
|
||||
if degradations is not None:
|
||||
degradations.append(refusal.degradation())
|
||||
continue
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch:
|
||||
trial = MechanismStore(Path(scratch) / "trial.jsonl")
|
||||
trial.append(record)
|
||||
recovered = [case.name for case in missed if score_pair_with_store(case, trial, facts).detected]
|
||||
leaks = [case.name for case in cases if not score_pair_with_store(case, trial, facts).silent]
|
||||
if recovered and not leaks:
|
||||
edits.append(
|
||||
MechanismEdit(
|
||||
action="admit",
|
||||
mechanism_id=record.id,
|
||||
source="loo",
|
||||
rationale=f"recovers holdout {', '.join(recovered)}; taught by {', '.join(record.pairs)}",
|
||||
)
|
||||
)
|
||||
return edits
|
||||
|
||||
|
||||
def _facts() -> SemanticFacts:
|
||||
from ..semantic.facts import FactLoadError, load_facts
|
||||
|
||||
try:
|
||||
return load_facts()
|
||||
except FactLoadError:
|
||||
return SemanticFacts(version="", sources=(), sinks=(), sanitizers=())
|
||||
|
||||
|
||||
def _leaking_admissions(edits: Sequence[MechanismEdit], pair_cases: Sequence[object], store_path: Path) -> list[str]:
|
||||
"""Admitted record ids that hit the fixed side of any holdout pair with the store as it stands after the edits."""
|
||||
from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored
|
||||
from ..semantic.loo import score_pair_with_store
|
||||
from ..semantic.mechanisms import MechanismStore
|
||||
|
||||
admitted = {edit.mechanism_id for edit in edits if edit.action == "admit"}
|
||||
if not admitted:
|
||||
return []
|
||||
cases = select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)
|
||||
store = MechanismStore(store_path)
|
||||
facts = _facts()
|
||||
leaking: set[str] = set()
|
||||
for case in cases:
|
||||
leaking |= set(score_pair_with_store(case, store, facts).leaked_by) & admitted
|
||||
return sorted(leaking)
|
||||
|
||||
|
||||
def profile_regressions(
|
||||
before: dict[str, dict[str, float]],
|
||||
after: dict[str, dict[str, float]],
|
||||
@@ -278,9 +159,6 @@ def run_round(
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
mechanism_candidates: Path | None = None,
|
||||
mechanism_store: Path | None = None,
|
||||
scripted_mechanism_edits: Sequence[MechanismEdit] | None = None,
|
||||
) -> RoundOutcome:
|
||||
policy = policy if policy is not None else load_policy()
|
||||
rounds = load_journal(journal_path)
|
||||
@@ -295,46 +173,18 @@ def run_round(
|
||||
|
||||
ruleset_by_id = {rule.rule_id: rule for rule in before_rules}
|
||||
edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
|
||||
# corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit.
|
||||
mechanism_edits: list[MechanismEdit] = []
|
||||
refusals: list[dict[str, object]] = []
|
||||
candidates_store = None
|
||||
if mechanism_candidates is not None and mechanism_store is not None and pair_cases:
|
||||
from ..semantic.mechanisms import MechanismStore
|
||||
|
||||
candidates_store = MechanismStore(mechanism_candidates)
|
||||
mechanism_edits = (
|
||||
list(scripted_mechanism_edits)
|
||||
if scripted_mechanism_edits is not None
|
||||
else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store, degradations=refusals)
|
||||
)
|
||||
blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked
|
||||
mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms]
|
||||
if not edits and not mechanism_edits:
|
||||
if not edits:
|
||||
return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals)
|
||||
|
||||
validator = EvolveValidator()
|
||||
try:
|
||||
for edit in edits:
|
||||
validator.validate(edit, ruleset_by_id, policy)
|
||||
if mechanism_edits and candidates_store is not None and mechanism_store is not None:
|
||||
from ..semantic.mechanisms import MechanismStore
|
||||
|
||||
by_id = {record.id: record for record in candidates_store.load()}
|
||||
admitted_by_id = {record.id: record for record in MechanismStore(mechanism_store).load()}
|
||||
for mechanism_edit in mechanism_edits:
|
||||
validator.validate_mechanism(mechanism_edit, by_id, admitted=admitted_by_id)
|
||||
except StrictValidationError as exc:
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits)
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, str(exc))
|
||||
return _outcome(
|
||||
round_index,
|
||||
accepted=False,
|
||||
reason=f"validation_failed: {exc}",
|
||||
edits=edits,
|
||||
before=before,
|
||||
after=before,
|
||||
mechanism_edits=mechanism_edits,
|
||||
degradations=refusals,
|
||||
round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before, degradations=refusals
|
||||
)
|
||||
|
||||
candidate = edits_to_ledger(edits, current_ledger)
|
||||
@@ -365,39 +215,10 @@ def run_round(
|
||||
regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs)
|
||||
if regressed:
|
||||
accepted = False
|
||||
mechanisms_before: dict[str, dict[str, float]] = {}
|
||||
mechanisms_after: dict[str, dict[str, float]] = {}
|
||||
mechanism_reason = ""
|
||||
if accepted and mechanism_edits and candidates_store is not None and mechanism_store is not None and pair_cases:
|
||||
# The lever changes what variant search finds, so its gate is the same per-profile clause over the holdout pairs
|
||||
# scored with the store before and after; an admission that leaks or lowers any profile is reverted byte for byte.
|
||||
mechanisms_before = evaluate_mechanism_profiles(pair_cases, mechanism_store)
|
||||
snapshot = apply_mechanism_edits(mechanism_edits, candidates_store, mechanism_store)
|
||||
mechanisms_after = evaluate_mechanism_profiles(pair_cases, mechanism_store)
|
||||
mech_regressed, mech_small = profile_regressions(
|
||||
mechanisms_before, mechanisms_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs
|
||||
)
|
||||
leaking = _leaking_admissions(mechanism_edits, pair_cases, mechanism_store)
|
||||
improved = any(
|
||||
mechanisms_after[p]["pair_correct"] > mechanisms_before.get(p, {}).get("pair_correct", 0.0) for p in mechanisms_after
|
||||
) or any(edit.action == "retract" for edit in mechanism_edits)
|
||||
if mech_regressed or leaking or not improved:
|
||||
snapshot.restore()
|
||||
accepted = False
|
||||
regressed = regressed + mech_regressed
|
||||
if leaking:
|
||||
mechanism_reason = f"mechanism_leak:{leaking[0]}" # an admission that fires on a holdout fixed side (Req 5.3)
|
||||
elif mech_regressed:
|
||||
mechanism_reason = f"profile_regression:{mech_regressed[0]}"
|
||||
else:
|
||||
mechanism_reason = "mechanism_no_gain"
|
||||
small = sorted(set(small) | set(mech_small))
|
||||
if accepted:
|
||||
if edits:
|
||||
write_rule_ledger(ledger_path, candidate)
|
||||
reason = "accepted"
|
||||
elif mechanism_reason:
|
||||
reason = mechanism_reason
|
||||
elif regressed:
|
||||
reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions
|
||||
else:
|
||||
@@ -412,20 +233,9 @@ def run_round(
|
||||
reason,
|
||||
regressed,
|
||||
small,
|
||||
mechanism_edits=mechanism_edits,
|
||||
mechanisms_before=mechanisms_before,
|
||||
mechanisms_after=mechanisms_after,
|
||||
candidates=candidates_store,
|
||||
)
|
||||
outcome = _outcome(
|
||||
round_index,
|
||||
accepted=accepted,
|
||||
reason=reason,
|
||||
edits=edits,
|
||||
before=before,
|
||||
after=after,
|
||||
mechanism_edits=mechanism_edits,
|
||||
degradations=refusals,
|
||||
round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, degradations=refusals
|
||||
)
|
||||
return RoundOutcome(
|
||||
**{
|
||||
@@ -434,8 +244,6 @@ def run_round(
|
||||
"profiles_under_minimum": small,
|
||||
"per_profile_before": profiles_before,
|
||||
"per_profile_after": profiles_after,
|
||||
"mechanisms_before": mechanisms_before,
|
||||
"mechanisms_after": mechanisms_after,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -454,8 +262,6 @@ def run_improvement(
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
mechanism_candidates: Path | None = None,
|
||||
mechanism_store: Path | None = None,
|
||||
) -> list[RoundOutcome]:
|
||||
"""Run improvement rounds until convergence (no new proposals) or ``max_rounds``."""
|
||||
outcomes: list[RoundOutcome] = []
|
||||
@@ -472,8 +278,6 @@ def run_improvement(
|
||||
pair_cases=pair_cases,
|
||||
profile_tolerance=profile_tolerance,
|
||||
min_holdout_pairs=min_holdout_pairs,
|
||||
mechanism_candidates=mechanism_candidates,
|
||||
mechanism_store=mechanism_store,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if outcome.reason == "no_proposals":
|
||||
@@ -542,7 +346,6 @@ def _outcome(
|
||||
edits: list[RuleStatusEdit],
|
||||
before: _Eval,
|
||||
after: _Eval,
|
||||
mechanism_edits: list[MechanismEdit] | None = None,
|
||||
degradations: list[dict[str, object]] | None = None,
|
||||
) -> RoundOutcome:
|
||||
return RoundOutcome(
|
||||
@@ -550,7 +353,6 @@ def _outcome(
|
||||
accepted=accepted,
|
||||
reason=reason,
|
||||
edits=edits,
|
||||
mechanism_edits=list(mechanism_edits or []),
|
||||
degradations=list(degradations or []),
|
||||
recall_before=before.recall,
|
||||
recall_after=after.recall,
|
||||
@@ -573,14 +375,7 @@ def _record(
|
||||
reason: str = "",
|
||||
profile_regressions: list[str] | None = None,
|
||||
profiles_under_minimum: list[str] | None = None,
|
||||
mechanism_edits: Sequence[MechanismEdit] = (),
|
||||
mechanisms_before: dict[str, dict[str, float]] | None = None,
|
||||
mechanisms_after: dict[str, dict[str, float]] | None = None,
|
||||
candidates: object | None = None,
|
||||
) -> None:
|
||||
pairs_by_id: dict[str, list[str]] = {}
|
||||
if candidates is not None:
|
||||
pairs_by_id = {record.id: list(record.pairs) for record in candidates.load()} # type: ignore[attr-defined]
|
||||
append_round(
|
||||
journal_path,
|
||||
{
|
||||
@@ -592,21 +387,7 @@ def _record(
|
||||
"edits": [
|
||||
{"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale}
|
||||
for e in edits
|
||||
]
|
||||
+ [
|
||||
{
|
||||
"key": m.key(),
|
||||
"lever": m.lever,
|
||||
"action": m.action,
|
||||
"mechanism_id": m.mechanism_id,
|
||||
"source": m.source,
|
||||
"pairs": pairs_by_id.get(m.mechanism_id, []),
|
||||
"rationale": m.rationale,
|
||||
}
|
||||
for m in mechanism_edits
|
||||
],
|
||||
"mechanisms_before": mechanisms_before or {},
|
||||
"mechanisms_after": mechanisms_after or {},
|
||||
"recall_before": round(before.recall, 4),
|
||||
"recall_after": round(after.recall, 4),
|
||||
"fp_before": round(before.fp_rate, 4),
|
||||
|
||||
Reference in New Issue
Block a user