mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
feat(contributor-scan): 1.3 the model layer reaches a user
`model-grounded-detection` finished 16/16 with the model layer having zero
production callers. This closes that. A real scan of a Flask file now reports:
by_rung: {"model_entailed": 3, "model_corroborated": 0, "suspicion": 0}
model:injection:ping:10 <- os.system("ping -c 1 " + request.args["host"])
The stage is the fifth MAP proposer, mirroring _check_obligations, and is
additive: pattern, overlay and obligation findings all keep working, and without
a CPG engine it records cpg_unavailable and changes nothing else. Two scan
integrity tests pin the exact degradation set and both broke -- correctly. Their
`requested == baseline` assertion, that findings are byte-identical, still
passed, so the only change is that the scan now SAYS the arbiter was absent
instead of being silently without one. The guards are updated to assert the
model degradation specifically rather than being loosened.
ModelLayerConfig, not ModelConfig: that name is already the [models] block, and
the type checker caught the collision immediately. Two meanings of one name is
how a scan silently reads the wrong settings.
Every payload carries the same shape, skipped ones included. A payload that is
sometimes a full record and sometimes a single "skipped" key is how a report
ends up quietly omitting a section.
unjudged_sample names the lowest-ranked regions the budget did not reach.
Counting them is the honesty requirement; naming them is what lets a maintainer
write a large-repository exclusion list from evidence rather than guesswork.
The run also produced the finding behind new task 1.4: 275 seconds for a
ten-line file. cost_usd is 0.0, so those 21 invocations were not model calls --
they were Joern query launches, one per region per family, each a JVM start.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
8663c1a5b3
commit
e67f3ee2f0
+128
-1
@@ -31,7 +31,7 @@ from .calibration import (
|
||||
)
|
||||
from .complexity import map as complexity_map
|
||||
from .complexity.ledger import persist_verdicts
|
||||
from .config import ObligationsConfig, load_config, load_dotenv
|
||||
from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, load_config, load_dotenv
|
||||
from .findings import StaticFinding, quick_scan_findings, write_findings
|
||||
from .fusion import FusionDecision, fuse_findings_dispatch
|
||||
from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
@@ -390,6 +390,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
overlay_records: list[OverlayRecord] = []
|
||||
wrote_overlay = False
|
||||
obligations_payload: dict[str, object] | None = None
|
||||
model_payload: dict[str, object] | None = None
|
||||
obligations_cited: dict[str, dict[str, object]] = {}
|
||||
if Stage.MAP in plan.requested:
|
||||
built_map = runtime.run_stage(
|
||||
@@ -436,6 +437,19 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
if obligation_findings:
|
||||
findings = rank_obligations(findings + obligation_findings)
|
||||
write_findings(findings, findings_path)
|
||||
# Fifth MAP proposer: the model layer (contributor-scan Req 1). Additive -- it adds findings beside
|
||||
# the pattern, overlay and obligation ones and removes none of them.
|
||||
model_findings, model_payload = _run_model_layer(
|
||||
root=run.target,
|
||||
targets=targets,
|
||||
entries=entry_points,
|
||||
settings=config.model,
|
||||
runtime=runtime,
|
||||
config=config,
|
||||
)
|
||||
if model_findings:
|
||||
findings = findings + model_findings
|
||||
write_findings(findings, findings_path)
|
||||
plan = record_completed(plan, Stage.MAP)
|
||||
if not hunter_model:
|
||||
runtime.state["degradations"].append(skip_as_degradation(Stage.MAP, "hunter_model_unavailable"))
|
||||
@@ -554,6 +568,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
worth_fixing=worth_fixing_payload,
|
||||
provenance=provenance.to_dict(),
|
||||
obligations=obligations_payload,
|
||||
model=model_payload,
|
||||
),
|
||||
)
|
||||
runtime.finish(status="succeeded")
|
||||
@@ -788,6 +803,118 @@ def _pairs(
|
||||
return 0
|
||||
|
||||
|
||||
def _model_skipped(reason: str) -> dict[str, object]:
|
||||
"""A skipped stage reports the same SHAPE as a run one, plus why it skipped.
|
||||
|
||||
A consumer reading this should not have to branch on whether the stage ran; a payload that is sometimes
|
||||
a full record and sometimes a single key is how a report ends up silently omitting a section.
|
||||
"""
|
||||
return {**_model_payload(None), "skipped": reason}
|
||||
|
||||
|
||||
def _model_payload(result: object, *, unjudged_paths: tuple[str, ...] = ()) -> dict[str, object]:
|
||||
"""What the scan records about the model layer.
|
||||
|
||||
`unjudged_sample` names the regions the budget did not reach, lowest-ranked first. Counting them is the
|
||||
honesty requirement (Req 3.3); NAMING them is what makes an exclusion list for a large repository
|
||||
something a maintainer can write from evidence rather than guesswork.
|
||||
"""
|
||||
return {
|
||||
"by_rung": dict(getattr(result, "by_rung", {}) or {}),
|
||||
"regions_scanned": getattr(result, "regions_scanned", 0),
|
||||
"regions_unjudged": getattr(result, "regions_unjudged", 0),
|
||||
"unjudged_sample": list(unjudged_paths),
|
||||
"model_calls": getattr(result, "model_calls", 0),
|
||||
"cost_usd": getattr(result, "cost_usd", 0.0),
|
||||
"seconds": getattr(result, "seconds", 0.0),
|
||||
}
|
||||
|
||||
|
||||
def _run_model_layer(
|
||||
*,
|
||||
root: Path,
|
||||
targets: list[FileTarget],
|
||||
entries: Sequence[object],
|
||||
settings: ModelLayerConfig,
|
||||
runtime: HarnessRuntime,
|
||||
config: ResolvedConfig,
|
||||
) -> tuple[list[StaticFinding], dict[str, object]]:
|
||||
"""Fifth MAP proposer (contributor-scan Req 1): the model layer, arbitrating regions of the repository.
|
||||
|
||||
Additive by construction. Without a CPG engine this records a reason and returns nothing, and the scan is
|
||||
byte-for-byte what it is today -- the model layer must never make the tool worse for someone who does not
|
||||
have Joern.
|
||||
"""
|
||||
if not settings.enabled:
|
||||
return [], _model_skipped("disabled")
|
||||
|
||||
from .cpg.backend import resolve_cpg_backend
|
||||
from .model.endpoint import resolve_chat_endpoint, resolve_models
|
||||
from .model.regions import regions_for
|
||||
from .model.scan import ScanBudget, scan_repository
|
||||
|
||||
backend = resolve_cpg_backend()
|
||||
if not backend.available():
|
||||
runtime.state["degradations"].append({"stage": "model", "reason": "cpg_unavailable"})
|
||||
return [], _model_skipped("cpg_unavailable")
|
||||
|
||||
regions = regions_for(entries, targets)
|
||||
if not regions:
|
||||
return [], _model_skipped("no_regions")
|
||||
|
||||
resolved = resolve_chat_endpoint(config)
|
||||
client = resolved[0] if resolved else None
|
||||
if client is None:
|
||||
# Not a failure: entailment still works and only the suspicion band goes unasked (Req 11.3).
|
||||
runtime.state["degradations"].append({"stage": "model", "reason": "learning_endpoint_unavailable"})
|
||||
|
||||
result = runtime.run_stage(
|
||||
"model",
|
||||
lambda: scan_repository(
|
||||
root,
|
||||
regions,
|
||||
backend=backend,
|
||||
client=client,
|
||||
model=resolve_models(config)[0],
|
||||
budget=ScanBudget(max_model_calls=settings.max_model_calls, max_regions=settings.max_regions),
|
||||
),
|
||||
)
|
||||
for degradation in result.degradations:
|
||||
runtime.state["degradations"].append(dict(degradation))
|
||||
|
||||
# The regions the budget did not reach, weakest first -- the evidence an exclusion list is built from.
|
||||
unjudged = tuple(region.path for region in regions[result.regions_scanned :][:10])
|
||||
findings = [_finding_from_model(item) for item in result.findings]
|
||||
return findings, _model_payload(result, unjudged_paths=unjudged)
|
||||
|
||||
|
||||
def _finding_from_model(item: object) -> StaticFinding:
|
||||
"""A model finding as a StaticFinding, carrying its rung and witness (Req 5.1)."""
|
||||
site = str(getattr(item, "site", ""))
|
||||
path, _, rest = site.partition(":")
|
||||
line = rest.split(":")[0] if rest else ""
|
||||
rung = getattr(item, "rung", None)
|
||||
witness = str(getattr(item, "witness", "") or getattr(item, "contradiction", ""))
|
||||
family = str(getattr(item, "family", ""))
|
||||
return StaticFinding(
|
||||
finding_id=f"model:{family}:{site}",
|
||||
path=path or site,
|
||||
title=f"{family} ({getattr(rung, 'value', rung)})",
|
||||
severity="medium",
|
||||
confidence="medium",
|
||||
evidence_level="suspicion",
|
||||
rationale=witness or f"the model layer reported a {family} candidate",
|
||||
line=int(line) if line.isdigit() else None,
|
||||
function_name=None,
|
||||
reachability_status="unknown",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[f"family:{family}", f"rung:{getattr(rung, 'value', rung)}"],
|
||||
ranking_priority=0.0,
|
||||
rung=str(getattr(rung, "value", rung) or "suspicion"),
|
||||
)
|
||||
|
||||
|
||||
def _check_obligations(
|
||||
*,
|
||||
root: Path,
|
||||
|
||||
@@ -143,6 +143,18 @@ class VariantsConfig:
|
||||
max_mechanisms: int = 500
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelLayerConfig:
|
||||
# contributor-scan: the model layer in MAP. Named for the LAYER, not the model: `ModelConfig` is already
|
||||
# the `[models]` block (hunter, judge, endpoint) and two meanings of one name is how you get a scan that
|
||||
# silently reads the wrong settings. Additive -- without a CPG engine the scan is unchanged.
|
||||
enabled: bool = True
|
||||
# A TOTAL for the run, not a per-region cap. Conservative by default: a first scan of an unfamiliar
|
||||
# repository should not be able to run away with someone's budget before they have decided they want it.
|
||||
max_model_calls: int = 100
|
||||
max_regions: int = 300
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObligationsConfig:
|
||||
# authorization-obligations: the obligation checker in MAP (standard/deep only).
|
||||
@@ -168,6 +180,7 @@ class ResolvedConfig:
|
||||
regress: RegressConfig = RegressConfig()
|
||||
variants: VariantsConfig = VariantsConfig()
|
||||
obligations: ObligationsConfig = ObligationsConfig()
|
||||
model: ModelLayerConfig = ModelLayerConfig()
|
||||
runs_dir: str = ".openultrasast/runs"
|
||||
|
||||
|
||||
@@ -194,6 +207,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig:
|
||||
regress=_load_regress(data.get("regress", {})),
|
||||
variants=_load_variants(data.get("variants", {})),
|
||||
obligations=_load_obligations(data.get("obligations", {})),
|
||||
model=_load_model(data.get("model", {})),
|
||||
runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"),
|
||||
)
|
||||
|
||||
@@ -369,6 +383,15 @@ def _load_variants(value: object) -> VariantsConfig:
|
||||
)
|
||||
|
||||
|
||||
def _load_model(value: object) -> ModelLayerConfig:
|
||||
data = _section(value)
|
||||
return ModelLayerConfig(
|
||||
enabled=bool(data.get("enabled", ModelLayerConfig.enabled)),
|
||||
max_model_calls=_int_value(data.get("max_model_calls"), ModelLayerConfig.max_model_calls),
|
||||
max_regions=_int_value(data.get("max_regions"), ModelLayerConfig.max_regions),
|
||||
)
|
||||
|
||||
|
||||
def _load_obligations(value: object) -> ObligationsConfig:
|
||||
data = _section(value)
|
||||
return ObligationsConfig(
|
||||
|
||||
@@ -205,6 +205,7 @@ def write_manifest(
|
||||
provenance: dict[str, object] | None = None,
|
||||
variants: dict[str, object] | None = None,
|
||||
obligations: dict[str, object] | None = None,
|
||||
model: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
verification_by_id = _verification_by_id(verifications)
|
||||
@@ -246,6 +247,8 @@ def write_manifest(
|
||||
payload["variants"] = variants
|
||||
if obligations is not None:
|
||||
payload["obligations"] = obligations
|
||||
if model is not None:
|
||||
payload["model"] = model
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user