mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): memory store (file and MinIO) and the remember task (harnessx-removal task 2)
plane/memory.py: the MemoryStore interface with FileStore (index.jsonl, facts by content hash, repos/<host>__<owner>__<name>/<pin>.jsonl, idempotent ingest, deterministic row ids, refusal below 1 GiB free naming the path) and MinioStore (object metadata and tags, S3 Select pushdown probed once per process with a logged local fallback, object version ids on every row read, versioning and runs/ lifecycle helper, presigned URLs). OUSAST_MEMORY selects the backend; MINIO_* settings come from .env; the minio SDK is the optional extra `minio`. plane/tasks/remember.py: model-free rows (facts, verdict, unit_cost, alert) from a case's artifacts; remember_run computes them on the host for a recorded run without writing into it. reconciler.mark_done (477 lines total), seed before and ingest after `ousast plane run`, `ousast plane remember <run>`. The generator annotates each repo-facts Task with openultrasast.io/memory-key (repo, pin, candidates digest, image digest) and the Run with its population and split; plane/ regenerated. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
d64ab2817e
commit
52d28b5121
@@ -0,0 +1,171 @@
|
||||
"""`remember` (harnessx-removal §4) on a recorded Run: a few KB of validation-46's lmdeploy case under
|
||||
``tests/fixtures/plane-remember`` (witness texts cut, facts.json cut to the files its callers name, the repository
|
||||
URL replaced: the population is reserved, ``tests/test_independent_population.py``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.plane import memory
|
||||
from openultrasast.plane.memory import FileStore, MemoryStoreError, ingest, row_id
|
||||
from openultrasast.plane.tasks import remember
|
||||
from openultrasast.plane.tasks.remember import Context, candidates_digest, remember_run, rows_for
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||
CASE = "lmdeploy-media-url-ssrf"
|
||||
PIN = "71648d239c978452720a6d6692fe914efc8a7b94"
|
||||
IMAGE = "sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "plane" / "validation-46"
|
||||
shutil.copytree(FIXTURE, target)
|
||||
return target
|
||||
|
||||
|
||||
def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
|
||||
rows: dict[str, list[dict[str, Any]]] = {}
|
||||
for record in store.rows():
|
||||
rows.setdefault(record.row["kind"], []).append(record.row)
|
||||
return rows
|
||||
|
||||
|
||||
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
|
||||
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
assert len(facts) > 100, "the fixture was read"
|
||||
store = FileStore(tmp_path / "memory")
|
||||
results = remember_run(run_dir, store, population="population-v2", split="validation")
|
||||
assert [(r.task, r.skipped, r.kinds) for r in results] == [(f"{CASE}-remember", False, {"facts": 1, "unit_cost": 2, "verdict": 1})]
|
||||
rows = _by_kind(store)
|
||||
for row in (r for kind in rows.values() for r in kind):
|
||||
assert (row["repo"], row["pin"], row["run"], row["image"]) == ("example.com/reserved/lmdeploy", PIN, "validation-46", IMAGE)
|
||||
assert (row["task"], row["population"], row["split"]) == (f"{CASE}-remember", "population-v2", "validation")
|
||||
sha = hashlib.sha256(facts).hexdigest()
|
||||
assert rows["facts"][0]["sha256"] == sha and store.get_facts(sha) == facts
|
||||
assert rows["facts"][0]["candidates_digest"] == candidates_digest(json.loads(facts))
|
||||
agreed = json.loads((run_dir / f"{CASE}-final" / "agreed.json").read_text())["candidates"]
|
||||
assert [(v["candidate"], v["passes"], v["site_match"], v["usd"], v["turns"]) for v in rows["verdict"]] == [
|
||||
(c["candidate"], {"a": c["a"], "b": c["b"], "c": c["c"]}, c["site_match"], c["usd"], c["turns"]) for c in agreed
|
||||
], "a verdict row per agree row, from the final (tie-break) decision"
|
||||
assert rows["verdict"][0]["final"] == "agreed" and rows["verdict"][0]["family"] == "untrusted_destination"
|
||||
costs = {(u["pass"], u["path"]): (u["usd"], u["calls"], u["model"]) for u in rows["unit_cost"]}
|
||||
assert costs == {
|
||||
("a", "lmdeploy/vl/media/connection.py"): (0.01237569, 7, "deepseek-flash"),
|
||||
("b", "lmdeploy/vl/media/connection.py"): (costs[("b", "lmdeploy/vl/media/connection.py")][0], 7, "deepseek-flash"),
|
||||
}
|
||||
assert sum(u["usd"] for u in rows["unit_cost"]) == pytest.approx(rows["verdict"][0]["usd"], abs=1e-6)
|
||||
assert not list(run_dir.glob("*-remember")), "the by-hand path writes nothing into the run directory"
|
||||
|
||||
|
||||
def test_re_ingest_of_the_same_run_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
remember_run(run_dir, store)
|
||||
before = {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()}
|
||||
again = remember_run(run_dir, store)
|
||||
assert [r.skipped for r in again] == [True]
|
||||
assert {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()} == before
|
||||
|
||||
|
||||
def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
agreed = {
|
||||
"family": "injection",
|
||||
"candidates": [
|
||||
{"candidate": "a.py::f", "a": True, "b": True, "c": None, "agreed": True},
|
||||
{"candidate": "a.py::g", "a": True, "b": False, "c": True, "agreed": True},
|
||||
{"candidate": "a.py::h", "a": True, "b": False, "c": False, "agreed": False},
|
||||
{"candidate": "a.py::i", "a": False, "b": False, "c": None, "agreed": False},
|
||||
],
|
||||
"disputed": [{"candidate": "a.py::h"}],
|
||||
}
|
||||
alert = {"rule_id": "py-ssrf", "rule_status": "shadow", "path": "a.py", "line": 3, "function": "f", "pin_role": "vulnerable"}
|
||||
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert])
|
||||
verdicts = {r["candidate"]: (r["final"], r["tiebreak"]) for r in rows if r["kind"] == "verdict"}
|
||||
expected = {"a.py::f": ("agreed", False), "a.py::g": ("agreed", True), "a.py::h": ("disputed", True), "a.py::i": ("rejected", False)}
|
||||
assert verdicts == expected
|
||||
(alert_row,) = [r for r in rows if r["kind"] == "alert"]
|
||||
assert {k: alert_row[k] for k in alert} == alert and alert_row["repo"] == "github.com/o/r"
|
||||
assert alert_row["id"] == row_id("alert", "r", "c-remember", "py-ssrf:a.py:3")
|
||||
assert len({r["id"] for r in rows}) == len(rows)
|
||||
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
|
||||
|
||||
|
||||
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
|
||||
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
|
||||
task = next(d for d in docs if d["kind"] == "Task")
|
||||
pins = next(e["value"] for e in task["spec"]["env"] if e["name"] == "OUSAST_GIT_PINS")
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_RUN": "validation-46", "OUSAST_TASK": f"{CASE}-remember", "OUSAST_GIT_PINS": pins,
|
||||
"AX_TASK_YAML": yaml.safe_dump(task), "AX_WORKSPACES_YAML": yaml.safe_dump_all([d for d in docs if d["kind"] == "Workspace"]),
|
||||
"OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"),
|
||||
"OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"),
|
||||
"OUSAST_POPULATION": "population-v2", "OUSAST_SPLIT": "validation",
|
||||
**{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"},
|
||||
**{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"},
|
||||
} # fmt: skip
|
||||
assert remember.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["units_done"], summary["usd"], summary["calls"], summary["rows"]) == ("done", 1, 0, 0, 4)
|
||||
assert (out / "facts.json").read_bytes() == (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
delivered = FileStore(tmp_path / "delivered")
|
||||
assert [r.task for r in ingest(run_dir, delivered)] == [f"{CASE}-remember"]
|
||||
by_hand = FileStore(tmp_path / "by-hand")
|
||||
(out / "memory.jsonl").rename(tmp_path / "memory.jsonl")
|
||||
remember_run(run_dir, by_hand, population="population-v2", split="validation")
|
||||
assert [r.row for r in delivered.rows()] == [r.row for r in by_hand.rows()], "the task and the host path agree"
|
||||
assert remember.main({**env, "AX_TASK_YAML": "{}"}) == 2
|
||||
assert "names no image" in json.loads((out / "summary.json").read_text())["reason"]
|
||||
|
||||
|
||||
def test_ingest_without_remember_outputs_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
assert ingest(run_dir, store) == [] and not store.root.exists()
|
||||
|
||||
|
||||
def test_a_malformed_delivered_row_fails_naming_file_and_line(run_dir: Path, tmp_path: Path) -> None:
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
out.mkdir()
|
||||
good = rows_for(Context("validation-46", out.name, "https://github.com/o/r", PIN, IMAGE), facts=None, passes={}, models={}, agreed=None,
|
||||
alerts=[{"rule_id": "x", "path": "a.py", "line": 1}]) # fmt: skip
|
||||
(out / "memory.jsonl").write_text(json.dumps(good[0]) + "\n" + json.dumps({**good[0], "kind": "vibes"}) + "\n")
|
||||
store = FileStore(tmp_path / "memory")
|
||||
with pytest.raises(MemoryStoreError, match=rf"{out.name}/memory.jsonl:2: unknown kind"):
|
||||
remember_run(run_dir, store)
|
||||
assert not store.root.exists()
|
||||
|
||||
|
||||
def test_plane_remember_cli_reads_population_from_the_run_and_reports_counts(
|
||||
run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "1 tasks ingested, 0 already in the index; rows facts=1 unit_cost=2 verdict=1" in out
|
||||
rows = FileStore(tmp_path / "memory").rows()
|
||||
assert {(r.row["population"], r.row["split"]) for r in rows} == {("population-v2", "validation")} # plane/runs/validation-46.yaml
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
assert "0 tasks ingested, 1 already in the index" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_plane_run_seeds_before_and_ingests_after(run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.plane import reconciler
|
||||
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(memory, "seed", lambda manifest, store: calls.append(f"seed {manifest.name}") or [])
|
||||
monkeypatch.setattr(reconciler, "run", lambda manifest, **kw: calls.append("run") or "done")
|
||||
assert cli.main(["plane", "run", "plane/runs/validation-46.yaml"]) == 0
|
||||
assert calls == ["seed validation-46.yaml", "run"]
|
||||
assert len(FileStore(tmp_path / "memory").rows(kind="verdict")) == 1, "the run's rows were ingested after it"
|
||||
Reference in New Issue
Block a user