marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 03:59:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(learning-harness): family detectors, verifiers and tags in a scan

Browse Source 
Task 3.4. A standard scan now classifies each hotspot region and runs the
detectors of every admitted family plus the generalist, reading the
configurations round zero wrote. A scan never creates them: a missing directory
is a recorded reason and the stage is skipped, which is also what happens when
no chat endpoint resolves.

Every claim passes through its family's verifier and carries what produced it:
the family, the detector version and what the verifier said. Nothing rises above
suspicion here, because the verifier writes a tag and the evidence ladder
belongs to another boundary. Reports gain a section naming the three, SARIF
gains the matching properties, and the manifest gains a learning block with the
taxonomy version, the per-family counts, the classifier report and the round.

The feature is off unless a project turns it on, so an existing scan is
unchanged, and all three gate outputs stay byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 11:33:26 +02:00
co-authored by Claude Opus 5
parent e8e6bafa61
commit 127dbef8fc
4 changed files with 229 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/cli.py
+99
View File
@@ -430,6 +430,8 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
variant_findings: list[StaticFinding] = []
mechanisms_cited: dict[str, dict[str, object]] = {}
obligations_payload: dict[str, object] | None = None
learning_payload: dict[str, object] | None = None
learning_cited: dict[str, dict[str, object]] = {}
obligations_cited: dict[str, dict[str, object]] = {}
if Stage.MAP in plan.requested:
built_map = runtime.run_stage(
@@ -489,6 +491,18 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
if obligation_findings:
findings = rank_obligations(findings + obligation_findings)
write_findings(findings, findings_path)
if config.learning.enabled:
family_findings, learning_payload, learning_cited = _run_family_detectors(
root=run.target,
hotspots=built_map.hotspots if built_map is not None else (),
cases=None,
settings=config.learning,
config=config,
runtime=runtime,
)
if family_findings:
findings = findings + family_findings
write_findings(findings, findings_path)
plan = record_completed(plan, Stage.MAP)
if not hunter_model:
runtime.state["degradations"].append(skip_as_degradation(Stage.MAP, "hunter_model_unavailable"))
@@ -592,6 +606,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
mechanisms=mechanisms_cited or None,
obligations=obligations_cited or None,
obligations_summary=obligations_payload or None,
learning=learning_cited or None,
),
)
runtime.run_stage(
@@ -603,6 +618,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
overlay=overlay_records if wrote_overlay else None,
mechanisms=mechanisms_cited or None,
obligations=obligations_cited or None,
learning=learning_cited or None,
),
)
runtime.run_stage(
@@ -633,6 +649,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
provenance=provenance.to_dict(),
variants=variants_payload,
obligations=obligations_payload,
learning=learning_payload,
),
)
runtime.finish(status="succeeded")
@@ -1063,6 +1080,88 @@ def _check_obligations(
return statics, payload, cited
def _run_family_detectors(
*,
root: Path,
hotspots: Sequence[object],
cases: object,
settings: object,
config: object,
runtime: HarnessRuntime,
) -> tuple[list[StaticFinding], dict[str, object], dict[str, dict[str, object]]]:
"""Fifth MAP proposer (learning-harness Req 6.3, 7.1): one detector per admitted family, plus the generalist.
Every claim passes through its family's verifier and carries what produced it. Nothing rises above
suspicion here: the verifier writes a tag, and the evidence ladder is another boundary's to move.
"""
del cases
from .learning.classify import classify_region, measure_classifier
from .learning.detectors import DetectorConfigError, Region, load_family_configs, run_region_detectors
from .learning.endpoint import resolve_chat_endpoint, resolve_models
from .learning.families import FamiliesError, load_families
from .learning.verifiers import apply_verification, verifier_for
empty: dict[str, dict[str, object]] = {}
configs_dir = root / str(getattr(settings, "configs_dir", ".openultrasast/learning/configs"))
try:
taxonomy = load_families(Path(p) if (p := getattr(settings, "families_path", None)) else None)
configs = load_family_configs(configs_dir, taxonomy)
except (FamiliesError, DetectorConfigError) as exc:
runtime.state["degradations"].append({"stage": "learning", "reason": "learning_configs_unavailable", "detail": str(exc)[:200]})
return [], {}, empty
if not configs:
runtime.state["degradations"].append(
{"stage": "learning", "reason": "learning_configs_unavailable", "detail": f"no family configurations under {configs_dir}"}
)
return [], {}, empty
resolved = resolve_chat_endpoint(config) # type: ignore[arg-type]
if resolved is None:
runtime.state["degradations"].append({"stage": "learning", "reason": "learning_endpoint_unavailable"})
return [], {}, empty
client, endpoint = resolved
model = resolve_models(config)[0] # type: ignore[arg-type]
findings: list[StaticFinding] = []
per_family: dict[str, int] = {}
seen: set[str] = set()
for hotspot in hotspots:
path = str(getattr(hotspot, "path", ""))
function = getattr(hotspot, "function_name", None)
key = f"{path}::{function}"
if not path or key in seen:
continue
seen.add(key)
answer = classify_region(root, path, function=function, taxonomy=taxonomy, client=None)
region = Region(path=path, function=str(function) if function else None, families=answer.families)
def _run(region: Region = region) -> list[StaticFinding]:
return run_region_detectors(root, region, configs, client=client, model=model)
produced = runtime.run_stage("learning", _run)
for finding in produced:
family_id = next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), "unknown")
verification = verifier_for(taxonomy.by_id(family_id)).verify(finding, root=root, language="python")
findings.append(apply_verification(finding, verification))
per_family[family_id] = per_family.get(family_id, 0) + 1
report = measure_classifier([], taxonomy, client=None)
payload: dict[str, object] = {
"taxonomy_version": taxonomy.version,
"families": per_family,
"classifier": report.to_dict(),
"round": None,
"endpoint": endpoint.provider,
"regions": len(seen),
}
cited: dict[str, dict[str, object]] = {
finding.finding_id: {
"family": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), ""),
"detector": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("detector:")), ""),
"verifier": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("verifier:")), ""),
}
for finding in findings
}
return findings, payload, cited
def _hotspot_from_variant(finding: StaticFinding) -> Hotspot:
return hotspot_from_finding(finding, rationale="variant of a known mechanism (suspicion; sandbox may raise)")
src/openultrasast/config.py
+2
View File
@@ -141,6 +141,7 @@ class VariantsConfig:
@dataclass(frozen=True)
class LearningConfig:
# learning-harness: the classified detector set and its rounds.
enabled: bool = False # off until round zero has written the family configurations this reads
families_path: str | None = None # taxonomy file; the shipped ruleset when unset
configs_dir: str = ".openultrasast/learning/configs"
k_runs: int = 5 # runs per pair; a single run misses most real changes
@@ -381,6 +382,7 @@ def _load_variants(value: object) -> VariantsConfig:
def _load_learning(value: object) -> LearningConfig:
data = _section(value)
return LearningConfig(
enabled=bool(data.get("enabled", LearningConfig.enabled)),
families_path=_string(data.get("families_path")),
configs_dir=_string(data.get("configs_dir")) or LearningConfig.configs_dir,
k_runs=_int_value(data.get("k_runs"), LearningConfig.k_runs),
src/openultrasast/reports.py
+23
View File
@@ -22,6 +22,7 @@ def write_markdown_report(
mechanisms: Mapping[str, Mapping[str, object]] | None = None,
obligations: Mapping[str, Mapping[str, object]] | None = None,
obligations_summary: Mapping[str, object] | None = None,
learning: Mapping[str, Mapping[str, object]] | None = None,
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
verification_by_id = _verification_by_id(verifications or [])
@@ -72,6 +73,8 @@ def write_markdown_report(
_append_mechanisms(lines, cited)
if cited_obligations:
_append_obligations(lines, cited_obligations, obligations_summary)
if learning:
_append_learning(lines, learning)
if overlay:
_append_overlay(lines, overlay)
if complexity_map is not None:
@@ -155,6 +158,7 @@ def write_sarif_report(
overlay: Sequence[object] | None = None,
mechanisms: Mapping[str, Mapping[str, object]] | None = None,
obligations: Mapping[str, Mapping[str, object]] | None = None,
learning: Mapping[str, Mapping[str, object]] | None = None,
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
verification_by_id = _verification_by_id(verifications)
@@ -179,6 +183,7 @@ def write_sarif_report(
overlay_by_id.get(finding.finding_id),
mechanisms=mechanisms,
obligation=(obligations or {}).get(finding.finding_id),
learning=(learning or {}).get(finding.finding_id),
)
for finding in findings
],
@@ -204,6 +209,7 @@ def write_manifest(
provenance: dict[str, object] | None = None,
variants: dict[str, object] | None = None,
obligations: dict[str, object] | None = None,
learning: dict[str, object] | None = None,
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
verification_by_id = _verification_by_id(verifications)
@@ -244,6 +250,8 @@ def write_manifest(
payload["variants"] = variants
if obligations is not None:
payload["obligations"] = obligations
if learning:
payload["learning"] = learning
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
@@ -287,6 +295,7 @@ def _sarif_result(
*,
mechanisms: Mapping[str, Mapping[str, object]] | None = None,
obligation: Mapping[str, object] | None = None,
learning: Mapping[str, object] | None = None,
) -> dict[str, object]:
properties: dict[str, object] = {
"finding_id": finding.finding_id,
@@ -318,6 +327,10 @@ def _sarif_result(
properties["obligation_evidence"] = _strings_of(obligation, "evidence")
properties["obligation_known_fix"] = obligation.get("known_fix")
properties["obligation_intent"] = obligation.get("intent")
if learning is not None:
properties["family"] = str(learning.get("family", ""))
properties["detector"] = str(learning.get("detector", ""))
properties["verifier"] = str(learning.get("verifier", ""))
return {
"ruleId": _rule_id(finding),
"level": _sarif_level(finding.severity),
@@ -473,6 +486,16 @@ def _append_obligations(lines: list[str], cited: Mapping[str, Mapping[str, objec
lines.append("")
def _append_learning(lines: list[str], cited: Mapping[str, Mapping[str, object]]) -> None:
"""Which family found a claim, which detector version produced it, and what a verifier said about it."""
lines.extend(["## Classified detectors", "", "Findings from the per-family detectors, with what confirmed them.", ""])
for finding_id, info in sorted(cited.items()):
lines.append(
f"- `{finding_id}`: family `{info.get('family')}`, detector `{info.get('detector')}`, verifier `{info.get('verifier')}`"
)
lines.append("")
def _strings_of(info: Mapping[str, object], key: str) -> list[str]:
value = info.get(key)
return [str(item) for item in value] if isinstance(value, list | tuple) else []