mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
feat(learn): candidate feature schema and builders (decision-engine task 1)
learn/schema.py is the closed, typed allow-list of candidate features: 66 features over 11 instruments (language one-hot, quick rules with 23 mechanism buckets and a prior count, engine, repo-facts, source span, entry-point distance, roles, delta, verify, agree, model sinks), two profiles (static 57; plane adds verify, agree and model sinks for 66), schema_version and a feature-set digest. No repository, path, file, function, rule or commit identity is a feature; site_match, in_fix_range, fixed-pin alerts, case.json fields and costs are refused. Missing is not zero: each record carries per-instrument state (ran | none | failed | not_applicable), and every feature of an instrument that did not run is null. learn/features.py builds records on the host (build_for_scan, whole repository or base..head delta); plane/tasks/features.py builds them per case in a Run from the delivered artifacts, dropping site_match and in_fix_range on read and ignoring fixed-pin alerts. The generator adds a model-free `features` entry per case before `remember`, which stores the records as `features` rows (re-validated). memory KINDS gains features, label, decision, experiment, arm_outcome and experiment_result. The committed validation-46 manifests are regenerated (entries appended; the first Run's entries are unchanged). The runner image must be rebuilt and re-pinned before a Run executes the new task. Proof on real artifacts (scratch store, not committed): the recorded validation-46 run gives 75 feature records over 15 cases (43 verify candidates plus vulnerable-pin alerts), each case's vulnerable pin exported and read (0.5k-9k files); remember stored 75 features rows beside 43 verdict, 84 alert, 61 unit_cost, 15 facts and 8 coverage rows. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
edcf145b00
commit
d1896cb8e7
@@ -15,7 +15,8 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
|
||||
(``triage.json``, for the measurement);
|
||||
- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
|
||||
Workspaces and naming the files as env;
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> remember``,
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> features ->
|
||||
remember`` (``features``: one allow-listed feature record per candidate, learned-decision-engine task 1),
|
||||
with budgets, annotated with the population and split (``openultrasast.io/population``/``split``: the files'
|
||||
stems) the memory rows carry.
|
||||
|
||||
@@ -86,7 +87,7 @@ CASE_PATH = "/workspace/case"
|
||||
INPUTS_PATH = "/workspace/inputs"
|
||||
FIXED_PATH = "/workspace/fixed"
|
||||
PROJECT_PATH = "/workspace/project"
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop")
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop", "features")
|
||||
FIXED_RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, new side: the fix's lines at the fixed pin"
|
||||
LOOP_MANIFEST = "benchmarks/manifests/java-spring-boot-vulnerable.toml" # the equality baseline's `improve` manifest
|
||||
LOOP_CATALOG = "benchmarks/pairs/catalog.toml"
|
||||
@@ -302,12 +303,9 @@ def _free() -> dict[str, int]:
|
||||
return {"usd": 0, "calls": 0} # model-free: budget.py refuses any call, so a stray one fails loudly
|
||||
|
||||
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
|
||||
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
|
||||
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
def _chain_inputs(case_id: str, alerts: bool) -> dict[str, str]:
|
||||
"""Every artifact of a case's chain -- facts, the three passes with their summaries, the final agree -- and, in a
|
||||
loop, its alerts with their coverage."""
|
||||
inputs = {"facts": f"{case_id}-facts/facts.json", "facts_summary": f"{case_id}-facts/summary.json"}
|
||||
for label in ("a", "b", "c"):
|
||||
inputs.update({f"pass_{label}": f"{case_id}-v{label}/units.jsonl", f"summary_{label}": f"{case_id}-v{label}/summary.json"})
|
||||
@@ -315,6 +313,28 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str,
|
||||
if alerts:
|
||||
inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl"
|
||||
inputs["alerts_summary"] = f"{case_id}-alerts/summary.json" # the coverage: whether a zero could mean clean
|
||||
return inputs
|
||||
|
||||
|
||||
def _features(item: CaseInputs, templates: Mapping[str, Task], alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `features` (learned-decision-engine design section 1): model-free, bound to the
|
||||
vulnerable Workspace for the functions' spans and entry-point distance; the chain's artifacts as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"}
|
||||
task = _task(templates["features"], f"features-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = _chain_inputs(case_id, alerts)
|
||||
entry = {"name": f"{case_id}-features", "task": f"features-{case_id}", "inputs": inputs, "outputs": ["features.jsonl", "summary.json"]}
|
||||
return task, {**entry, "budget": _free()}
|
||||
|
||||
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
|
||||
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) and the case's
|
||||
feature records as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
|
||||
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = {**_chain_inputs(case_id, alerts), "features": f"{case_id}-features/features.jsonl"}
|
||||
entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()}
|
||||
return task, entry
|
||||
|
||||
@@ -382,7 +402,7 @@ def render(
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
|
||||
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
|
||||
case ends in `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
|
||||
case ends in `features` -> `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
|
||||
snapshot, `loop-measure`, `loop-propose` and `loop-improve` (design section 6), all model-free."""
|
||||
header = f"# generated by `{command}`; do not edit, regenerate.\n"
|
||||
out: dict[str, str] = {}
|
||||
@@ -435,6 +455,9 @@ def render(
|
||||
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
|
||||
tasks.append(alerts_task)
|
||||
entries.append(alerts_entry)
|
||||
features_task, features_entry = _features(item, templates, loop is not None)
|
||||
tasks.append(features_task)
|
||||
entries.append(features_entry)
|
||||
remember_task, remember_entry = _remember(item, templates, population, split, loop is not None)
|
||||
tasks.append(remember_task)
|
||||
entries.append(remember_entry)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4).
|
||||
|
||||
One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows
|
||||
of six kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``proposal_outcome``) that ``remember``
|
||||
(:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends::
|
||||
of the kinds in ``KINDS`` (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``features``, ...) that
|
||||
``remember`` (:mod:`.tasks.remember`) derives from a run's delivered artifacts, plus the loop's and the decision
|
||||
engine's own. Layout, identical for both backends::
|
||||
|
||||
index.jsonl one row per ingested (run, task): sha256 of its rows, row count
|
||||
facts/<sha256>.json facts.json by content (identical facts stored once)
|
||||
@@ -49,7 +50,12 @@ MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
|
||||
POPULATION_ANNOTATION = "openultrasast.io/population"
|
||||
SPLIT_ANNOTATION = "openultrasast.io/split"
|
||||
SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"}
|
||||
KINDS = ("facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome")
|
||||
# The decision engine's kinds (learned-decision-engine, Data Models): `features` from the `features` task through
|
||||
# `remember`; `label`, `decision`, `experiment`, `arm_outcome` and `experiment_result` for the later tasks.
|
||||
KINDS = (
|
||||
"facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome",
|
||||
"features", "label", "decision", "experiment", "arm_outcome", "experiment_result",
|
||||
) # fmt: skip
|
||||
ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
|
||||
TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
|
||||
MIN_FREE_BYTES = 1 << 30
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""`features` (learned-decision-engine design section 1): one feature record per candidate of a case. No model.
|
||||
|
||||
Budget ``{usd: 0, calls: 0}``, one per case, after ``final`` (and ``alerts`` in a loop Run). A candidate is a
|
||||
``(path, function, family)`` that the verify passes asked about or a vulnerable-pin alert named. It reads:
|
||||
|
||||
- ``facts.json`` (repo-facts callers), the verify passes' ``units.jsonl`` a, b, c and their ``summary.json`` (the
|
||||
model id is the verify instrument's version), the final ``agreed.json`` -- with ``site_match`` dropped on read;
|
||||
- in a loop, ``alerts.jsonl`` -- vulnerable-pin rows only, ``in_fix_range`` dropped: a deployed scan has no fixed pin
|
||||
and no fix -- and the alerts' ``summary.json`` (coverage per language, the engine's questions and degradations);
|
||||
- the bound checkout (``OUSAST_WORKSPACE_DIR``) for the function's span and its entry-point distance.
|
||||
|
||||
It writes ``features.jsonl`` (records validated by :func:`..learn.schema.validate_record`, the `plane` profile) and
|
||||
``summary.json``; ``remember`` turns the records into ``features`` rows. An instrument without coverage for the
|
||||
candidate's language is ``none``, one that did not read its input ``failed``: never a zero (Req 1.2). The task fails
|
||||
when every instrument it could have used failed.
|
||||
|
||||
Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_INPUT_FACTS`` (and ``OUSAST_INPUT_FACTS_SUMMARY``), ``OUSAST_INPUT_PASS_A``/``_B``/``_C``,
|
||||
``OUSAST_INPUT_SUMMARY_A``/``_B``/``_C``, ``OUSAST_INPUT_AGREED``, optional ``OUSAST_INPUT_ALERTS`` and
|
||||
``OUSAST_INPUT_ALERTS_SUMMARY``, optional ``OUSAST_WORKSPACE_DIR``. Exit 0 done, 2 failed. On the host,
|
||||
:func:`case_features` does the same over a recorded run directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ...learn.features import (
|
||||
NONE,
|
||||
NOT_APPLICABLE,
|
||||
Part,
|
||||
SourceIndex,
|
||||
Vocabulary,
|
||||
agree_part,
|
||||
engine_part,
|
||||
entry_distance,
|
||||
entry_names,
|
||||
facts_part,
|
||||
quick_part,
|
||||
record,
|
||||
ruleset_digest,
|
||||
source_part,
|
||||
verify_part,
|
||||
)
|
||||
from ...model.taxonomy import load_families
|
||||
from ...preprocess import detect_language
|
||||
from ...ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
||||
from .alerts import covers
|
||||
from .repo_facts import GLOBAL
|
||||
|
||||
OUTPUT = "features.jsonl"
|
||||
PASSES = ("a", "b", "c")
|
||||
ENGINE_PREFIX = "engine:"
|
||||
# Read, never passed on: label information (the declared site, the fix) and the fixed pin.
|
||||
DROPPED_ON_READ = ("site_match", "in_fix_range")
|
||||
|
||||
|
||||
def _drop(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in row.items() if k not in DROPPED_ON_READ}
|
||||
|
||||
|
||||
def _family(rule: PatternRule | None) -> str:
|
||||
family = load_families().family_of_cwe(rule.cwe) if rule is not None else None
|
||||
return family.id if family is not None else "unknown"
|
||||
|
||||
|
||||
def _line(site: object) -> int | None:
|
||||
parts = str(site or "").split(":")
|
||||
return int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else None
|
||||
|
||||
|
||||
def _final(row: Mapping[str, Any], disputed: set[str]) -> str:
|
||||
if row.get("agreed"):
|
||||
return "agreed"
|
||||
return "disputed" if str(row["candidate"]) in disputed else "rejected"
|
||||
|
||||
|
||||
def _quick(
|
||||
summary: Mapping[str, Any] | None, language: str, hits: list[tuple[str, str]], rules: Mapping[str, PatternRule], version: str
|
||||
) -> Part:
|
||||
if summary is None:
|
||||
return NONE
|
||||
if summary.get("status") not in (None, "done"):
|
||||
return Part("failed")
|
||||
if not covers(summary.get("quick_languages") or (), language):
|
||||
return NONE
|
||||
return quick_part(hits, rules, version)
|
||||
|
||||
|
||||
def _engine(summary: Mapping[str, Any] | None, language: str, rows: list[dict[str, Any]], vocabulary: Vocabulary) -> Part:
|
||||
if summary is None or language not in (summary.get("engine_languages") or ()):
|
||||
return NONE
|
||||
run = (summary.get("engine") or {}).get("vulnerable")
|
||||
if not isinstance(run, Mapping):
|
||||
return NONE
|
||||
questions, completed = int(run.get("questions") or 0), int(run.get("completed") or 0)
|
||||
if questions <= 0 or int(run.get("files") or 0) <= 0:
|
||||
return Part("failed", version=summary.get("image"))
|
||||
return engine_part(
|
||||
rows, completion=completed / questions, degraded=bool(run.get("degradations")), vocabulary=vocabulary, version=summary.get("image")
|
||||
)
|
||||
|
||||
|
||||
def case_features(
|
||||
*,
|
||||
facts: Mapping[str, Any] | None,
|
||||
passes: Mapping[str, Sequence[Mapping[str, Any]]],
|
||||
models: Mapping[str, str | None],
|
||||
agreed: Mapping[str, Any] | None,
|
||||
alerts: Sequence[Mapping[str, Any]] = (),
|
||||
alerts_summary: Mapping[str, Any] | None = None,
|
||||
workspace: Path | None = None,
|
||||
rules: Sequence[PatternRule] | None = None,
|
||||
facts_summary: Mapping[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The `plane` records of one case, sorted by (candidate, family). Versions: the ruleset digest (quick), the
|
||||
engine image (engine), the image the facts were computed on (facts), the verify model ids (verify)."""
|
||||
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
||||
by_id = {rule.rule_id: rule for rule in loaded}
|
||||
candidates: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
verdicts = [_drop(c) for c in (agreed or {}).get("candidates") or []]
|
||||
disputed = {str(d["candidate"]) for d in (agreed or {}).get("disputed") or []}
|
||||
for row in verdicts:
|
||||
family = str((row.get("finding") or {}).get("family") or (agreed or {}).get("family") or "unknown")
|
||||
entry = candidates.setdefault((str(row["candidate"]), family), {"line": _line(row.get("site")), "quick": [], "engine": []})
|
||||
entry["final"] = _final(row, disputed)
|
||||
for alert in (_drop(a) for a in alerts):
|
||||
if alert.get("pin_role") != "vulnerable":
|
||||
continue # a deployed scan has no fixed pin
|
||||
rule_id = str(alert.get("rule_id") or "")
|
||||
engine = rule_id.startswith(ENGINE_PREFIX)
|
||||
family = rule_id.removeprefix(ENGINE_PREFIX) if engine else _family(by_id.get(rule_id))
|
||||
key = (f"{alert.get('path')}::{alert.get('function') or GLOBAL}", family)
|
||||
entry = candidates.setdefault(key, {"line": alert.get("line"), "quick": [], "engine": []})
|
||||
if engine:
|
||||
entry["engine"].append({"site": f"{alert.get('path')}:{alert.get('line')}"})
|
||||
else:
|
||||
entry["quick"].append((rule_id, str(alert.get("rule_status") or "enabled")))
|
||||
callers = (facts or {}).get("callers") or {}
|
||||
index = SourceIndex(workspace) if workspace is not None and workspace.is_dir() else None
|
||||
entries: tuple[set[str], set[str]] | None = None
|
||||
quick_version = ruleset_digest(loaded)
|
||||
vocabularies: dict[str, Vocabulary] = {}
|
||||
reused = (facts_summary or {}).get("reused")
|
||||
facts_image = (reused.get("image") if isinstance(reused, Mapping) else None) or (facts_summary or {}).get("image")
|
||||
facts_version = str(facts_image) if facts_image else None
|
||||
verify_version = "+".join(sorted({m for m in models.values() if m})) or None
|
||||
out: list[dict[str, Any]] = []
|
||||
for (candidate, family), entry in sorted(candidates.items()):
|
||||
path, _, function = candidate.partition("::")
|
||||
language = detect_language(Path(path))
|
||||
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
|
||||
parts: dict[str, Part] = {
|
||||
"quick": _quick(alerts_summary, language, entry["quick"], by_id, quick_version),
|
||||
"engine": _engine(alerts_summary, language, entry["engine"], vocabulary),
|
||||
"facts": facts_part(callers[candidate], function, facts_version) if candidate in callers else NONE,
|
||||
"delta": NOT_APPLICABLE,
|
||||
"verify": verify_part(passes, candidate, entry["line"], verify_version),
|
||||
"agree": agree_part(entry.get("final")),
|
||||
}
|
||||
if index is not None:
|
||||
if entries is None:
|
||||
entries = entry_names(index.root, {c.partition("::")[0] for c, _ in candidates})
|
||||
parts["source"] = source_part(index.lines(path), language, function)
|
||||
distance = 0 if function == GLOBAL and path in entries[1] else entry_distance(function, entries[0], index.callers())
|
||||
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
|
||||
out.append(record(candidate, family, language, "plane", parts))
|
||||
return out
|
||||
|
||||
|
||||
def _jsonl(path: Path | None) -> list[dict[str, Any]]:
|
||||
if path is None or not path.is_file():
|
||||
return []
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _json(path: Path | None) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path is not None and path.is_file() else None
|
||||
|
||||
|
||||
def _model(summary: object) -> str | None:
|
||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||
|
||||
|
||||
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
|
||||
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
|
||||
final = base / f"{case}-final" / "agreed.json"
|
||||
return case_features(
|
||||
facts=_json(base / f"{case}-facts" / "facts.json"),
|
||||
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
|
||||
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
|
||||
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
|
||||
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
|
||||
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
|
||||
)
|
||||
|
||||
|
||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
|
||||
def _units(path: Path | None) -> list[dict[str, Any]]:
|
||||
return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path)
|
||||
|
||||
|
||||
def _states(records: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, int]]:
|
||||
counts: dict[str, dict[str, int]] = {}
|
||||
for rec in records:
|
||||
for name, info in rec["instruments"].items():
|
||||
counts.setdefault(name, {}).setdefault(info["state"], 0)
|
||||
counts[name][info["state"]] += 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
env = environ if environ is not None else os.environ
|
||||
output_dir = _path(env, "OUSAST_OUTPUT_DIR")
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
try:
|
||||
if output_dir is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
workspace = _path(env, "OUSAST_WORKSPACE_DIR")
|
||||
records = case_features(
|
||||
facts=_json(_path(env, "OUSAST_INPUT_FACTS")),
|
||||
passes={p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES},
|
||||
models={p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES},
|
||||
agreed=_json(_path(env, "OUSAST_INPUT_AGREED")),
|
||||
alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")),
|
||||
alerts_summary=_json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY")),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(_path(env, "OUSAST_INPUT_FACTS_SUMMARY")),
|
||||
)
|
||||
states = _states(records)
|
||||
used = {name: n for name, n in states.items() if name not in ("language", "delta") and set(n) - {"none"}}
|
||||
if records and used and all(set(n) == {"failed"} for n in used.values()):
|
||||
raise ValueError(f"every instrument failed for this case: {used}")
|
||||
(output_dir / OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in records), encoding="utf-8")
|
||||
summary.update(status="done", units_done=1, records=len(records), instruments=states)
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
if output_dir is not None:
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: summary.get(k) for k in ("status", "records")}), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the
|
||||
verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present,
|
||||
`alerts.jsonl` with the alerts' `summary.json`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`,
|
||||
`unit_cost`, `alert` and `coverage` (per language and pin: whether any rule could have fired there; the kind
|
||||
`alerts.jsonl` with the alerts' `summary.json`, and the `features` task's `features.jsonl`, and writes `memory.jsonl`
|
||||
-- rows of the kinds `facts`, `verdict`, `unit_cost`, `alert`, `coverage` (per language and pin: whether any rule
|
||||
could have fired there) and `features` (one per candidate and family, validated against the allow-list; the kind
|
||||
`proposal_outcome` comes from the loop) -- and passes `facts.json` through. Task code interprets the
|
||||
artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4).
|
||||
|
||||
@@ -14,7 +15,7 @@ Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST
|
||||
whose reused or recorded `image` the facts row carries; without one named, the row is omitted),
|
||||
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
|
||||
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS` (and `OUSAST_INPUT_ALERTS_SUMMARY`,
|
||||
the coverage); the repository and pin from the bound
|
||||
the coverage), `OUSAST_INPUT_FEATURES`; the repository and pin from the bound
|
||||
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
|
||||
`OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and
|
||||
`openultrasast.io/split` annotations). Exit 0 done, 2 failed.
|
||||
@@ -38,6 +39,7 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ...learn.schema import validate_record
|
||||
from ..memory import REMEMBER_OUTPUT, IngestResult, MemoryStore, image_digest, ingest, repo_key, row_id, validate_row
|
||||
from .repo_facts import _digest
|
||||
|
||||
@@ -105,8 +107,10 @@ def rows_for(
|
||||
agreed: Mapping[str, Any] | None,
|
||||
alerts: Sequence[Mapping[str, Any]] = (),
|
||||
alerts_summary: Mapping[str, Any] | None = None,
|
||||
features: Sequence[Mapping[str, Any]] = (),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The memory rows of one case: facts, verdicts, unit costs, alerts, and the alerts' coverage per language and pin."""
|
||||
"""The memory rows of one case: facts, verdicts, unit costs, alerts, the alerts' coverage per language and pin, and
|
||||
the feature records (re-validated: a delivered record outside the allow-list fails the case)."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
if facts is not None:
|
||||
loaded = json.loads(facts)
|
||||
@@ -143,6 +147,9 @@ def rows_for(
|
||||
subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "")
|
||||
rows.append(ctx.row("alert", subject, **fields))
|
||||
rows.extend(coverage_rows(ctx, alerts_summary))
|
||||
for rec in features:
|
||||
checked = validate_record(rec)
|
||||
rows.append(ctx.row("features", f"{checked['candidate']}:{checked['family']}", **checked))
|
||||
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
|
||||
|
||||
|
||||
@@ -212,7 +219,10 @@ def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[li
|
||||
models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES}
|
||||
alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl")
|
||||
alerts_summary = _json(base / f"{case}-alerts" / "summary.json")
|
||||
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
|
||||
features = _jsonl(base / f"{case}-features" / "features.jsonl")
|
||||
rows = rows_for(
|
||||
ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary, features=features
|
||||
)
|
||||
return rows, facts
|
||||
|
||||
|
||||
@@ -275,7 +285,10 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
|
||||
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
|
||||
alerts, alerts_summary = _jsonl(_path(env, "OUSAST_INPUT_ALERTS")), _json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY"))
|
||||
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
|
||||
features = _jsonl(_path(env, "OUSAST_INPUT_FEATURES"))
|
||||
rows = rows_for(
|
||||
ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary, features=features
|
||||
)
|
||||
facts_image = _facts_image(env, ctx.image)
|
||||
if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on
|
||||
facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split)
|
||||
|
||||
Reference in New Issue
Block a user