From 52d28b5121257a1b0093408b134fd8511d25dc06 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:10:28 +0200 Subject: [PATCH] feat(plane): memory store (file and MinIO) and the remember task (harnessx-removal task 2) plane/memory.py: the MemoryStore interface with FileStore (index.jsonl, facts by content hash, repos/____/.jsonl, idempotent ingest, deterministic row ids, refusal below 1 GiB free naming the path) and MinioStore (object metadata and tags, S3 Select pushdown probed once per process with a logged local fallback, object version ids on every row read, versioning and runs/ lifecycle helper, presigned URLs). OUSAST_MEMORY selects the backend; MINIO_* settings come from .env; the minio SDK is the optional extra `minio`. plane/tasks/remember.py: model-free rows (facts, verdict, unit_cost, alert) from a case's artifacts; remember_run computes them on the host for a recorded run without writing into it. reconciler.mark_done (477 lines total), seed before and ingest after `ousast plane run`, `ousast plane remember `. The generator annotates each repo-facts Task with openultrasast.io/memory-key (repo, pin, candidates digest, image digest) and the Run with its population and split; plane/ regenerated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/harnessx-removal/tasks.md | 2 +- .../measurements/2026-09-08-module-audit.json | 34 +- plane/runs/validation-46.yaml | 3 + plane/tasks/validation-46.yaml | 30 + pyproject.toml | 10 + src/openultrasast/cli.py | 44 ++ src/openultrasast/model/audit.py | 2 + src/openultrasast/plane/generate.py | 28 +- src/openultrasast/plane/memory.py | 695 ++++++++++++++++++ src/openultrasast/plane/reconciler.py | 14 +- src/openultrasast/plane/tasks/remember.py | 246 +++++++ .../lmdeploy-media-url-ssrf-agree/agreed.json | 59 ++ .../disputed.json | 1 + .../summary.json | 38 + .../lmdeploy-media-url-ssrf-facts/facts.json | 22 + .../summary.json | 9 + .../lmdeploy-media-url-ssrf-facts/task.yaml | 40 + .../lmdeploy-media-url-ssrf-final/agreed.json | 66 ++ .../disputed.json | 1 + .../summary.json | 44 ++ .../lmdeploy-media-url-ssrf-va/summary.json | 16 + .../lmdeploy-media-url-ssrf-va/units.jsonl | 1 + .../lmdeploy-media-url-ssrf-vb/summary.json | 16 + .../lmdeploy-media-url-ssrf-vb/units.jsonl | 1 + .../lmdeploy-media-url-ssrf-vc/summary.json | 17 + .../lmdeploy-media-url-ssrf-vc/units.jsonl | 0 tests/test_plane_increment.py | 11 +- tests/test_plane_memory.py | 383 ++++++++++ tests/test_plane_reconciler.py | 16 + tests/test_plane_remember.py | 171 +++++ uv.lock | 115 ++- 31 files changed, 2121 insertions(+), 14 deletions(-) create mode 100644 src/openultrasast/plane/memory.py create mode 100644 src/openultrasast/plane/tasks/remember.py create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/agreed.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/disputed.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/facts.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/task.yaml create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/agreed.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/disputed.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/units.jsonl create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/units.jsonl create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/summary.json create mode 100644 tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/units.jsonl create mode 100644 tests/test_plane_memory.py create mode 100644 tests/test_plane_remember.py diff --git a/.kiro/specs/harnessx-removal/tasks.md b/.kiro/specs/harnessx-removal/tasks.md index a7a1068..dd5d671 100644 --- a/.kiro/specs/harnessx-removal/tasks.md +++ b/.kiro/specs/harnessx-removal/tasks.md @@ -12,7 +12,7 @@ and passed its fixture test. Every task is a commit gated on the full suite's ow written next to it. - _Requirements: 3.1, 5.1_ -- [ ] 2. Memory store and the `remember` task +- [x] 2. Memory store and the `remember` task - `plane/memory.py`: the `MemoryStore` interface with `FileStore` and `MinioStore` (object metadata/tags, S3 Select JSON queries probed with a local fallback, versioned provenance, lifecycle, presign stubs; `OUSAST_MEMORY` selects; MinIO settings from `.env`; `minio` SDK as an optional extra); JSON rows per diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 325b8f1..f5b790a 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 132, + "modules": 134, "counts": { - "load_bearing": 123, + "load_bearing": 125, "standalone_capability": 9 }, "orphaned": [], @@ -117,6 +117,7 @@ "hunter_tools", "model.endpoint", "pairs", + "plane.memory", "push.assistance", "push.policy", "push.runner", @@ -663,9 +664,11 @@ "cli", "plane.egress", "plane.generate", + "plane.memory", "plane.reconciler", "plane.router", "plane.runner", + "plane.tasks.remember", "plane.tasks.verify", "plane.workspaces" ] @@ -716,12 +719,22 @@ "plane.workspaces" ] }, + { + "module": "plane.memory", + "classification": "load_bearing", + "reason": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`", + "importers": [ + "plane.generate", + "plane.tasks.remember" + ] + }, { "module": "plane.reconciler", "classification": "load_bearing", "reason": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`", "importers": [ "plane.generate", + "plane.memory", "plane.workspaces" ] }, @@ -745,8 +758,10 @@ "classification": "load_bearing", "reason": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)", "importers": [ + "cli", "plane.generate", - "plane.tasks.agree" + "plane.tasks.agree", + "plane.tasks.remember" ] }, { @@ -755,11 +770,22 @@ "reason": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match", "importers": [] }, + { + "module": "plane.tasks.remember", + "classification": "load_bearing", + "reason": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free", + "importers": [ + "cli" + ] + }, { "module": "plane.tasks.repo_facts", "classification": "load_bearing", "reason": "ai-service-plane Req 5.1/5.2: source-only functions and callers per file; runner integration pending", - "importers": [] + "importers": [ + "plane.generate", + "plane.tasks.remember" + ] }, { "module": "plane.tasks.verify", diff --git a/plane/runs/validation-46.yaml b/plane/runs/validation-46.yaml index 93ff97a..20f3977 100644 --- a/plane/runs/validation-46.yaml +++ b/plane/runs/validation-46.yaml @@ -4,6 +4,9 @@ apiVersion: openultrasast.io/v1alpha1 kind: Run metadata: name: validation-46 + annotations: + openultrasast.io/population: population-v2 + openultrasast.io/split: validation spec: tasks: - name: budibase-mongo-template-nosqli-facts diff --git a/plane/tasks/validation-46.yaml b/plane/tasks/validation-46.yaml index 82a5958..7fc8cee 100644 --- a/plane/tasks/validation-46.yaml +++ b/plane/tasks/validation-46.yaml @@ -4,6 +4,8 @@ kind: Task metadata: name: repo-facts-budibase-mongo-template-nosqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -120,6 +122,8 @@ kind: Task metadata: name: repo-facts-cct-studio-server-cmdi atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -236,6 +240,8 @@ kind: Task metadata: name: repo-facts-contextforge-prompt-ssti atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -352,6 +358,8 @@ kind: Task metadata: name: repo-facts-froxlor-admin-ip-second-order-sqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -468,6 +476,8 @@ kind: Task metadata: name: repo-facts-fuxa-tdengine-escape-sqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -584,6 +594,8 @@ kind: Task metadata: name: repo-facts-lmdeploy-media-url-ssrf atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -700,6 +712,8 @@ kind: Task metadata: name: repo-facts-medplum-external-auth-open-redirect atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -816,6 +830,8 @@ kind: Task metadata: name: repo-facts-pgadmin-maintenance-sqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -932,6 +948,8 @@ kind: Task metadata: name: repo-facts-qwed-sympy-eval atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1048,6 +1066,8 @@ kind: Task metadata: name: repo-facts-reactpress-findall-column-sqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1164,6 +1184,8 @@ kind: Task metadata: name: repo-facts-unleash-addon-webhook-ssrf atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1280,6 +1302,8 @@ kind: Task metadata: name: repo-facts-winml-cli-serve-cors atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1396,6 +1420,8 @@ kind: Task metadata: name: repo-facts-wp-directorist-avatar-ssrf atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1512,6 +1538,8 @@ kind: Task metadata: name: repo-facts-wp-gopay-log-filter-sqli atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: @@ -1628,6 +1656,8 @@ kind: Task metadata: name: repo-facts-wp-groundhogg-confirm-redirect atespace: default + annotations: + openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}' spec: image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617 command: diff --git a/pyproject.toml b/pyproject.toml index adfe474..8cdc29b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,11 @@ dependencies = ["pyyaml>=6.0"] harnessx = [ "harnessx @ git+https://github.com/Darwin-Agent/HarnessX.git@274c512eea67c64701f992d3b6fe06f0fd590ef0", ] +# The plane's memory store on MinIO (OUSAST_MEMORY=minio://, plane/memory.py). Imported lazily; the +# core install and the file store need nothing beyond PyYAML. +minio = [ + "minio>=7.2,<8", +] # Structured parse for overlay (JS/C/C++/Java/Python). Core install stays empty. # Wheels are MIT; import is lazy via openultrasast.semantic.extra. semantic = [ @@ -74,6 +79,11 @@ ignore_missing_imports = true module = "tree_sitter" ignore_missing_imports = true +# The minio SDK is an optional extra, absent in the type-check environment (plane/memory.py imports it lazily). +[[tool.mypy.overrides]] +module = ["minio", "minio.*"] +ignore_missing_imports = true + # PyYAML ships no inline types and types-PyYAML is not a dev dependency. [[tool.mypy.overrides]] module = "yaml" diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 20cc372..16b56d2 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -248,6 +248,11 @@ def main(argv: list[str] | None = None) -> int: plane_status.add_argument("run") plane_status.add_argument("--units", action="store_true", help="per-unit rows from each task's units.jsonl") plane_sub.add_parser("doctor", help="check kind, Agent Substrate, the ax controller and the runner image") + plane_remember = plane_sub.add_parser("remember", help="ingest a run's memory rows into the store (OUSAST_MEMORY)") + plane_remember.add_argument("run") + plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/.yaml names population and split") + plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)") + plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)") plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file") plane_ws.add_argument("population", type=Path) plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces")) @@ -1304,9 +1309,14 @@ def _plane(args: argparse.Namespace) -> int: from .plane import reconciler # lazy: the plane is a maintainer surface, not the scan path if args.plane_command == "run": + _plane_memory("seed", args.run_manifest) result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax) print(f"run finished: {result}") + _plane_memory("ingest", args.run_manifest) return 0 if result == "done" else 1 + if args.plane_command == "remember": + manifest = args.plane / "runs" / f"{args.run}.yaml" + return _plane_memory("ingest", manifest if manifest.is_file() else None, run=args.run, population=args.population, split=args.split) if args.plane_command == "status": print(reconciler.status(args.run, units=args.units)) return 0 @@ -1324,6 +1334,40 @@ def _plane(args: argparse.Namespace) -> int: return 0 +def _plane_memory( + step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None +) -> int: + """``seed`` before a Run, ``ingest`` after it (and ``ousast plane remember``). A store failure is reported and + never changes the run's result: ingest can be repeated by hand.""" + from .plane import memory, reconciler + from .plane.tasks.remember import UNKNOWN, remember_run + + try: + store = memory.open_store() + if step == "seed": + assert run_manifest is not None + seeded = memory.seed(run_manifest, store) + if seeded: + print(f"memory {store.describe()}: facts reused for {len(seeded)} tasks: {', '.join(seeded)}") + return 0 + annotations: dict[str, str] = {} + if run_manifest is not None: + spec, _ = reconciler.load_run(run_manifest) + run, annotations = run or spec.metadata.name, dict(spec.metadata.annotations) + assert run is not None + results = remember_run( + reconciler.run_dir(run), store, population=population or annotations.get(memory.POPULATION_ANNOTATION) or UNKNOWN, + split=split or annotations.get(memory.SPLIT_ANNOTATION) or UNKNOWN, + ) # fmt: skip + except (memory.MemoryStoreError, OSError, ValueError) as exc: + print(f"memory {step} failed: {exc}", file=sys.stderr) + return 1 + fresh = [r for r in results if not r.skipped] + kinds = " ".join(f"{k}={n}" for k, n in memory.counts(fresh).items()) or "none" + print(f"memory {store.describe()}: {run}: {len(fresh)} tasks ingested, {len(results) - len(fresh)} already in the index; rows {kinds}") + return 0 + + def _plane_increment(args: argparse.Namespace) -> int: from .plane.generate import increment diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index e4c2e0f..2a1bfe2 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -63,6 +63,8 @@ SPEC_OWNED = { "plane.tasks.agree": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match", "plane.reconciler": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`", "plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set", + "plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`", + "plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free", "plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`", "plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory", } diff --git a/src/openultrasast/plane/generate.py b/src/openultrasast/plane/generate.py index 517b08d..b755974 100644 --- a/src/openultrasast/plane/generate.py +++ b/src/openultrasast/plane/generate.py @@ -15,7 +15,11 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi (``triage.json``, for the measurement); - ``tasks/.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two Workspaces and naming the files as env; -- ``runs/.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets. +- ``runs/.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets, annotated with the + population and split (``openultrasast.io/population``/``split``: the files' stems) the memory rows carry. + +Each ``repo-facts`` Task carries ``openultrasast.io/memory-key`` (repository, pin, candidates digest, runner image +digest): the key under which stored facts are reused instead of recomputed (``plane/memory.py``, ``seed``). Why per-case Tasks rather than one Task per step: an ax Task binds its Workspaces by name and a Run entry adds neither a binding nor env (it names a Task, artifacts and a budget), and ``Run.inputs`` accepts only @@ -53,7 +57,9 @@ import yaml from ..model.endpoint import price_of from .budget import cost_of from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests +from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, memory_key from .reconciler import _ax_name, _doc +from .tasks.repo_facts import _digest from .tasks.verify import MAX_STEPS, units_of __all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"] @@ -251,8 +257,11 @@ def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]: ] # fmt: skip -def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str) -> dict[str, str]: - """Relative path under the plane root -> file text, for every generated manifest.""" +def render( + cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str, *, population: str = "", split: str = "" +) -> dict[str, str]: + """Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split`` + (the population file's and the set's stems) become the Run's annotations, which the memory rows carry.""" header = f"# generated by `{command}`; do not edit, regenerate.\n" out: dict[str, str] = {} tasks: list[dict[str, Any]] = [] @@ -266,7 +275,11 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: repo = [(pinned["metadata"]["name"], CASE_PATH), (inputs["metadata"]["name"], INPUTS_PATH)] on_repo = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"} facts_env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/functions.json"} - tasks.append(_task(templates["repo-facts"], f"repo-facts-{case_id}", facts_env, repo)) + facts_task = _task(templates["repo-facts"], f"repo-facts-{case_id}", facts_env, repo) + digest = _digest(sorted({fn for _, fn, _ in item.kept})) + key = memory_key(str(item.case["repo"]), str(item.case["vulnerable"]), digest, templates["repo-facts"].image or "") + facts_task["metadata"].setdefault("annotations", {})[MEMORY_KEY_ANNOTATION] = key + tasks.append(facts_task) for label in ("a", "b"): env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json", "OUSAST_PASS": label} tasks.append(_task(templates["verify"], f"verify-{label}-{case_id}", env, repo)) @@ -295,7 +308,9 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state out[f"tasks/{run_name}.yaml"] = _dump(header, tasks) - run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": {"name": run_name}, "spec": {"tasks": entries}} + annotations = {k: v for k, v in ((POPULATION_ANNOTATION, population), (SPLIT_ANNOTATION, split)) if v} + metadata: dict[str, Any] = {"name": run_name, **({"annotations": annotations} if annotations else {})} + run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": metadata, "spec": {"tasks": entries}} total = sum(float(e["budget"]["usd"]) for e in entries) candidates = sum(len(c.candidates) for c in cases) kept = sum(len(c.kept) for c in cases) @@ -340,7 +355,8 @@ def increment( if runner_image is not None: repin_templates(plane, runner_image) loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES]) - files = render(read_cases(validation_set, candidates_dir, triage_dir, population, repos), loaded.tasks, run_name, command) + cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos) + files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem) written: list[Path] = [] for relative, text in files.items(): path = plane / relative diff --git a/src/openultrasast/plane/memory.py b/src/openultrasast/plane/memory.py new file mode 100644 index 0000000..6e15863 --- /dev/null +++ b/src/openultrasast/plane/memory.py @@ -0,0 +1,695 @@ +"""The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4). + +One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows +of five kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``proposal_outcome``) that the ``remember`` task +(:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends:: + + index.jsonl one row per ingested (run, task): sha256 of its rows, row count + facts/.json facts.json by content (identical facts stored once) + repos/____/.jsonl the rows of one repository + 40-hex pin, sorted by id + +``OUSAST_MEMORY`` selects the backend: ``file:///path`` (:class:`FileStore`, default ``results_root()/memory``) or +``minio://[/]`` (:class:`MinioStore`, the ``minio`` extra). MinIO's endpoint and credentials come +from ``.env`` (``MINIO_ENDPOINT``, ``MINIO_ACCESS_KEY``, ``MINIO_SECRET_KEY``, ``MINIO_SECURE``), never from a +manifest, and are never printed. Ingest is idempotent: an ``index.jsonl`` hit on (run, task, sha256) skips the +delivery, and rows carry a deterministic ``id``, so a repeated row replaces itself. Every object is written whole +(``FileStore``: a temporary file renamed) and the index last, so an interrupted ingest leaves nothing partial that +a repeat would not repair. ``FileStore`` refuses to write below 1 GiB free and names its path. + +Fact reuse: a facts entry is valid for (repo, pin, candidates digest, runner image digest), the key the generator +writes as the Task annotation ``openultrasast.io/memory-key``. :func:`seed` runs before a Run: for every entry whose +Task carries a key with stored facts and no state yet, it writes ``//facts.json`` and a ``done`` +summary and marks the task done under the run lock (``reconciler.mark_done``), so the reconciler skips it. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import shutil +import tempfile +from abc import ABC, abstractmethod +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Protocol +from urllib.parse import urlsplit + +log = logging.getLogger(__name__) + +MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key" +POPULATION_ANNOTATION = "openultrasast.io/population" +SPLIT_ANNOTATION = "openultrasast.io/split" +KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome") +ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image") +TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split") +MIN_FREE_BYTES = 1 << 30 +MINIO_KEYS = ("MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY") +_PIN = re.compile(r"[0-9a-f]{40}") +_FIELD = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_TAG_VALUE = re.compile(r"[^A-Za-z0-9 +\-=._:/@]") + + +class MemoryStoreError(RuntimeError): + """The store refused or failed a read or write; the message names the store and the cause.""" + + +# --- rows and keys ------------------------------------------------------------------------------------------------- + + +def repo_key(repo: str) -> str: + """``host/owner/name`` of a repository URL (scheme, credentials, ``.git`` and trailing slashes dropped).""" + text = repo.strip() + if "://" in text: + parts = urlsplit(text) + text = f"{parts.hostname or ''}{parts.path}" + text = text.rstrip("/").removesuffix(".git").strip("/") + if not text: + raise ValueError(f"not a repository: {repo!r}") + return text + + +def repo_dir(repo: str) -> str: + return repo_key(repo).replace("/", "__") + + +def row_id(kind: str, run: str, task: str, subject: str) -> str: + """The deterministic row id: sha256 of (kind, run, task, subject), subject being the candidate, the path, + the facts sha256 or the rule id with its location.""" + return hashlib.sha256("\x00".join((kind, run, task, subject)).encode("utf-8")).hexdigest() + + +def image_digest(image: str) -> str: + """The content part of an image reference (``sha256:...``): the registry host may differ, the code may not.""" + return image.rpartition("@")[2] if "@" in image else image + + +def memory_key(repo: str, pin: str, candidates: str, image: str) -> str: + """The annotation text of a facts entry's validity key.""" + key = {"candidates": candidates, "image": image_digest(image), "pin": pin, "repo": repo_key(repo)} + return json.dumps(key, sort_keys=True, separators=(",", ":")) + + +def parse_memory_key(text: str) -> dict[str, str]: + key = json.loads(text) + if not isinstance(key, dict) or set(key) != {"candidates", "image", "pin", "repo"}: + raise ValueError(f"{MEMORY_KEY_ANNOTATION} must name candidates, image, pin and repo: {text[:200]!r}") + return {k: str(v) for k, v in key.items()} + + +def validate_row(row: object, where: str = "row") -> dict[str, Any]: + if not isinstance(row, dict): + raise MemoryStoreError(f"{where}: a row must be a JSON object") + missing = [f for f in ROW_FIELDS if not isinstance(row.get(f), str) or not row.get(f)] + if missing: + raise MemoryStoreError(f"{where}: missing or empty fields {missing}") + if row["kind"] not in KINDS: + raise MemoryStoreError(f"{where}: unknown kind {row['kind']!r} (one of {', '.join(KINDS)})") + if not _PIN.fullmatch(row["pin"]): + raise MemoryStoreError(f"{where}: pin {row['pin']!r} is not a 40-hex commit") + return row + + +def _line(row: Mapping[str, Any]) -> str: + return json.dumps(row, sort_keys=True, separators=(",", ":")) + + +def rows_digest(rows: Iterable[Mapping[str, Any]]) -> str: + return hashlib.sha256("".join(_line(r) + "\n" for r in rows).encode("utf-8")).hexdigest() + + +def matches(row: Mapping[str, Any], where: Mapping[str, Any] | None) -> bool: + """Top-level equality on every ``where`` field; the rule the S3 Select expression pushes down.""" + return all(row.get(k) == v for k, v in (where or {}).items()) + + +def where_sql(where: Mapping[str, Any]) -> str: + """The S3 Select expression of ``where`` (JSON Lines input): equality per field, AND-joined.""" + clauses: list[str] = [] + for name, value in sorted(where.items()): + if not _FIELD.fullmatch(name): + raise ValueError(f"where: field {name!r} is not a plain identifier") + ref = f's."{name}"' + if value is None: + clauses.append(f"{ref} IS NULL") + elif isinstance(value, bool): + clauses.append(f"{ref} = {'true' if value else 'false'}") + elif isinstance(value, int | float): + clauses.append(f"{ref} = {value!r}") + elif isinstance(value, str): + clauses.append(f"{ref} = '{value.replace(chr(39), chr(39) * 2)}'") + else: + raise ValueError(f"where: {name} must be a string, number, boolean or null") + return "SELECT * FROM S3Object s" + (" WHERE " + " AND ".join(clauses) if clauses else "") + + +@dataclass(frozen=True) +class Record: + """A row as read, with the object it came from and that object's version (Req 6.4 provenance).""" + + row: dict[str, Any] + key: str + version: str | None + + +@dataclass(frozen=True) +class IngestResult: + run: str + task: str + rows: int + skipped: bool + kinds: dict[str, int] + + +# --- the interface ------------------------------------------------------------------------------------------------- + + +class MemoryStore(ABC): + """Rows by repository and pin, facts by content hash, and the ingest index; two backends, one behaviour.""" + + # primitives a backend implements; keys are the layout's paths + @abstractmethod + def _get(self, key: str) -> tuple[bytes, str | None] | None: ... + + @abstractmethod + def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: ... + + @abstractmethod + def _delete(self, key: str) -> None: ... + + @abstractmethod + def _keys(self, prefix: str) -> list[str]: ... + + @abstractmethod + def describe(self) -> str: ... + + @abstractmethod + def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: ... + + @abstractmethod + def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: ... + + def _check_space(self) -> None: # noqa: B027 -- optional: only a backend with a local disk overrides it + """A backend with a local disk refuses a write that would fill it.""" + + def _select(self, key: str, where: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None] | None: + """Rows of ``key`` matching ``where``, filtered where the data lives; None when the object is absent.""" + found = self._get(key) + if found is None: + return None + return [r for r in _parse_lines(found[0], key) if matches(r, where)], found[1] + + def _object_may_match(self, key: str, kind: str | None) -> bool: + return True + + # --- reads --- + def index(self) -> list[dict[str, Any]]: + found = self._get("index.jsonl") + return _parse_lines(found[0], "index.jsonl") if found else [] + + def get_facts(self, sha256: str) -> bytes | None: + """The facts.json stored under ``sha256``; a file whose content no longer hashes to it is dropped with a + warning and reported absent, so the task recomputes.""" + key = f"facts/{sha256}.json" + found = self._get(key) + if found is None: + return None + if hashlib.sha256(found[0]).hexdigest() != sha256: + log.warning("memory %s: %s no longer matches its sha256; dropped, the task recomputes", self.describe(), key) + self._delete(key) + return None + return found[0] + + def rows( + self, repo: str | None = None, pin: str | None = None, kind: str | None = None, where: Mapping[str, Any] | None = None + ) -> list[Record]: + """Rows filtered by repository, pin, kind and top-level equality on ``where``, in key then id order.""" + prefix = f"repos/{repo_dir(repo)}/" if repo else "repos/" + keys = [k for k in self._keys(prefix) if k.endswith(".jsonl") and (pin is None or k.endswith(f"/{pin}.jsonl"))] + wanted = {**(where or {}), **({"kind": kind} if kind else {})} + records: list[Record] = [] + for key in sorted(keys): + if not self._object_may_match(key, kind): + continue + selected = self._select(key, wanted) + if selected is not None: + records.extend(Record(r, key, selected[1]) for r in sorted(selected[0], key=lambda r: str(r["id"]))) + return records + + # --- writes --- + def put_facts(self, data: bytes) -> str: + sha = hashlib.sha256(data).hexdigest() + key = f"facts/{sha}.json" + if self._get(key) is None: + self._check_space() + self._put(key, data) + return sha + + def put_row(self, row: Mapping[str, Any]) -> None: + self.put_rows([row]) + + def put_rows(self, rows: Sequence[Mapping[str, Any]]) -> None: + """Merge rows into their repository + pin objects by ``id`` (a repeated id replaces the stored row).""" + groups: dict[str, dict[str, dict[str, Any]]] = {} + for index, row in enumerate(rows): + checked = validate_row(dict(row), f"row {index}") + groups.setdefault(f"repos/{repo_dir(checked['repo'])}/{checked['pin']}.jsonl", {})[checked["id"]] = checked + if groups: + self._check_space() + for key, new in sorted(groups.items()): + found = self._get(key) + merged = {str(r["id"]): r for r in (_parse_lines(found[0], key) if found else [])} + merged.update(new) + ordered = [merged[i] for i in sorted(merged)] + self._put(key, "".join(_line(r) + "\n" for r in ordered).encode("utf-8"), _labels(ordered)) + + def ingest_rows(self, run: str, task: str, rows: Sequence[Mapping[str, Any]], facts: bytes | None = None) -> IngestResult: + """One (run, task)'s rows and facts; a repeat of the same rows is skipped by the index.""" + checked = [validate_row(dict(r), f"{run}/{task} row {i + 1}") for i, r in enumerate(rows)] + digest = rows_digest(checked) + kinds: dict[str, int] = {} + for row in checked: + kinds[row["kind"]] = kinds.get(row["kind"], 0) + 1 + index = self.index() + if any(e.get("run") == run and e.get("task") == task and e.get("sha256") == digest for e in index): + return IngestResult(run, task, len(checked), True, kinds) + self._check_space() + if facts is not None: + self.put_facts(facts) + self.put_rows(checked) + entry = {"run": run, "task": task, "sha256": digest, "rows": len(checked), "ingested": _now()} + self._put("index.jsonl", "".join(_line(e) + "\n" for e in [*index, entry]).encode("utf-8")) + return IngestResult(run, task, len(checked), False, kinds) + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds") + + +def _parse_lines(data: bytes, where: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for number, raw in enumerate(data.decode("utf-8").splitlines(), 1): + if not raw.strip(): + continue + try: + row = json.loads(raw) + except ValueError as exc: + raise MemoryStoreError(f"{where}:{number}: not JSON ({exc})") from exc + if not isinstance(row, dict): + raise MemoryStoreError(f"{where}:{number}: not a JSON object") + rows.append(row) + return rows + + +def _labels(rows: Sequence[Mapping[str, Any]]) -> dict[str, str]: + """Object metadata/tags: per field the distinct values of the object's rows, space-joined, S3-safe, <=256.""" + labels: dict[str, str] = {} + for name in TAG_FIELDS: + values = sorted({_TAG_VALUE.sub("_", str(r[name])) for r in rows if r.get(name) not in (None, "")}) + text = " ".join(values) + labels[name] = text if len(text) <= 256 else "*" # "*": too many to list; the object may match anything + return labels + + +# --- the file backend ---------------------------------------------------------------------------------------------- + + +class FileStore(MemoryStore): + """The layout under a directory: this laptop's store and the unit tests'.""" + + def __init__(self, root: Path) -> None: + self.root = Path(root) + + def describe(self) -> str: + return f"file://{self.root}" + + def _path(self, key: str) -> Path: + path = (self.root / key).resolve() + if not path.is_relative_to(self.root.resolve()): + raise MemoryStoreError(f"{key!r} escapes the store {self.root}") + return path + + def _check_space(self) -> None: + probe = self.root + while not probe.exists() and probe != probe.parent: + probe = probe.parent + free = shutil.disk_usage(probe).free + if free < MIN_FREE_BYTES: + raise MemoryStoreError( + f"refusing to write the memory store at {self.root}: {free / (1 << 30):.2f} GiB free, below 1 GiB " + "(free space, or set OUSAST_MEMORY to another store; `ousast plane remember ` repeats the ingest)" + ) + + def _get(self, key: str) -> tuple[bytes, str | None] | None: + path = self._path(key) + if not path.is_file(): + return None + data = path.read_bytes() + return data, "sha256:" + hashlib.sha256(data).hexdigest() # a file has no version: its content names it + + def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: + path = self._path(key) + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "wb") as sink: + sink.write(data) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + def _delete(self, key: str) -> None: + self._path(key).unlink(missing_ok=True) + + def _keys(self, prefix: str) -> list[str]: + base = self._path(prefix) + if not base.is_dir(): + return [] + return sorted(p.relative_to(self.root.resolve()).as_posix() for p in base.rglob("*") if p.is_file() and not p.name.startswith(".")) + + def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + return self._path(key).as_uri() + + def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + return self._path(key).as_uri() + + +# --- the MinIO backend --------------------------------------------------------------------------------------------- + + +class ObjectClient(Protocol): + """The object operations :class:`MinioStore` needs; :class:`SdkClient` adapts the ``minio`` SDK, tests fake it.""" + + def get(self, key: str) -> tuple[bytes, str | None] | None: ... + def put(self, key: str, data: bytes, labels: Mapping[str, str]) -> None: ... + def delete(self, key: str) -> None: ... + def keys(self, prefix: str) -> list[str]: ... + def tags(self, key: str) -> dict[str, str]: ... + def version(self, key: str) -> str | None: ... + def select(self, key: str, expression: str) -> bytes: ... + def presign(self, method: str, key: str, expires: timedelta) -> str: ... + def expire(self, prefix: str, days: int, rule_id: str) -> None: ... + def enable_versioning(self) -> None: ... + + +_SELECT_PROBES: dict[str, bool] = {} +PROBE_KEY = "_probe/select.jsonl" + + +class MinioStore(MemoryStore): + """The layout in one bucket (optionally under a prefix): object metadata and tags on every row object, S3 + Select pushdown when the server answers it (probed once per process) and a local filter otherwise, bucket + versioning for provenance, lifecycle expiry for ``runs/``, presigned URLs.""" + + def __init__(self, client: ObjectClient, bucket: str, prefix: str = "") -> None: + self.client, self.bucket, self.prefix = client, bucket, prefix.strip("/") + + def describe(self) -> str: + return f"minio://{self.bucket}" + (f"/{self.prefix}" if self.prefix else "") + + def _k(self, key: str) -> str: + return f"{self.prefix}/{key}" if self.prefix else key + + def _get(self, key: str) -> tuple[bytes, str | None] | None: + return self.client.get(self._k(key)) + + def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: + self.client.put(self._k(key), data, dict(labels or {})) + + def _delete(self, key: str) -> None: + self.client.delete(self._k(key)) + + def _keys(self, prefix: str) -> list[str]: + cut = len(self.prefix) + 1 if self.prefix else 0 + return sorted(k[cut:] for k in self.client.keys(self._k(prefix))) + + def _object_may_match(self, key: str, kind: str | None) -> bool: + """Filter by tag instead of reading: an object whose ``kind`` tag lacks ``kind`` holds no such row.""" + if kind is None: + return True + tagged = self.client.tags(self._k(key)).get("kind") + return tagged is None or tagged == "*" or kind in tagged.split() + + def select_supported(self) -> bool: + """Whether the server answers S3 Select over JSON Lines: one probe per bucket per process.""" + name = self.describe() + if name not in _SELECT_PROBES: + try: + self.client.put(self._k(PROBE_KEY), b'{"probe":1}\n{"probe":2}\n', {}) + got = _parse_lines(self.client.select(self._k(PROBE_KEY), where_sql({"probe": 1})), PROBE_KEY) + _SELECT_PROBES[name] = got == [{"probe": 1}] + detail = "" if _SELECT_PROBES[name] else f"; the probe returned {got!r}" + except Exception as exc: # noqa: BLE001 -- any failure means "not supported"; the fallback is exact + _SELECT_PROBES[name], detail = False, f"; {type(exc).__name__}: {str(exc)[:200]}" + if not _SELECT_PROBES[name]: + log.warning("memory %s: the server does not answer S3 Select%s; filtering rows locally", name, detail) + return _SELECT_PROBES[name] + + def _select(self, key: str, where: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None] | None: + if not where or not self.select_supported(): + return super()._select(key, where) + version = self.client.version(self._k(key)) # stat, then select: Select takes no version id + if version is None and self.client.get(self._k(key)) is None: + return None + try: + return _parse_lines(self.client.select(self._k(key), where_sql(where)), key), version + except Exception as exc: # noqa: BLE001 -- one object's Select failing never loses rows + log.warning("memory %s: S3 Select on %s failed (%s); filtering it locally", self.describe(), key, type(exc).__name__) + return super()._select(key, where) + + def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + return self.client.presign("PUT", self._k(key), expires) + + def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + return self.client.presign("GET", self._k(key), expires) + + def configure(self, runs_expire_days: int = 30) -> None: + """Bucket versioning on (provenance) and the lifecycle rule expiring ``runs/`` after ``runs_expire_days``; + ``repos/``, ``facts/`` and ``proposals/`` are kept.""" + if runs_expire_days < 1: + raise ValueError("runs_expire_days must be at least 1") + self.client.enable_versioning() + self.client.expire(self._k("runs/"), runs_expire_days, "ousast-runs-expiry") + + +class SdkClient: + """:class:`ObjectClient` over ``minio.Minio``; imported lazily so the core install needs no SDK.""" + + def __init__(self, endpoint: str, access_key: str, secret_key: str, secure: bool, bucket: str) -> None: + try: + from minio import Minio + except ImportError as exc: + raise MemoryStoreError("OUSAST_MEMORY=minio://... needs the minio SDK: install openultrasast[minio]") from exc + self.sdk, self.bucket = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure), bucket + + def get(self, key: str) -> tuple[bytes, str | None] | None: + from minio.error import S3Error + + try: + response = self.sdk.get_object(self.bucket, key) + except S3Error as exc: + if exc.code in ("NoSuchKey", "NoSuchObject"): + return None + raise + try: + return response.read(), response.headers.get("x-amz-version-id") + finally: + response.close() + response.release_conn() + + def put(self, key: str, data: bytes, labels: Mapping[str, str]) -> None: + import io + + from minio.commonconfig import Tags + + tags = Tags(for_object=True) + for name, value in labels.items(): + tags[name] = value + metadata: dict[str, str | list[str] | tuple[str]] = dict(labels) + self.sdk.put_object( + self.bucket, key, io.BytesIO(data), len(data), content_type="application/x-ndjson", + metadata=metadata or None, tags=tags if labels else None, + ) # fmt: skip + + def delete(self, key: str) -> None: + self.sdk.remove_object(self.bucket, key) + + def keys(self, prefix: str) -> list[str]: + return sorted(str(o.object_name) for o in self.sdk.list_objects(self.bucket, prefix=prefix, recursive=True)) + + def tags(self, key: str) -> dict[str, str]: + return dict(self.sdk.get_object_tags(self.bucket, key) or {}) + + def version(self, key: str) -> str | None: + from minio.error import S3Error + + try: + version = self.sdk.stat_object(self.bucket, key).version_id + return None if version is None else str(version) + except S3Error as exc: + if exc.code in ("NoSuchKey", "NoSuchObject"): + return None + raise + + def select(self, key: str, expression: str) -> bytes: + from minio.select import JSONInputSerialization, JSONOutputSerialization, SelectRequest + + request = SelectRequest(expression, JSONInputSerialization(json_type="LINES"), JSONOutputSerialization(), request_progress=False) + with self.sdk.select_object_content(self.bucket, key, request) as result: + return b"".join(result.stream()) + + def presign(self, method: str, key: str, expires: timedelta) -> str: + if method == "PUT": + return str(self.sdk.presigned_put_object(self.bucket, key, expires=expires)) + return str(self.sdk.presigned_get_object(self.bucket, key, expires=expires)) + + def expire(self, prefix: str, days: int, rule_id: str) -> None: + from minio.commonconfig import ENABLED, Filter + from minio.lifecycleconfig import Expiration, LifecycleConfig, Rule + + rule = Rule(ENABLED, rule_filter=Filter(prefix=prefix), rule_id=rule_id, expiration=Expiration(days=days)) + self.sdk.set_bucket_lifecycle(self.bucket, LifecycleConfig([rule])) + + def enable_versioning(self) -> None: + from minio.commonconfig import ENABLED + from minio.versioningconfig import VersioningConfig + + self.sdk.set_bucket_versioning(self.bucket, VersioningConfig(ENABLED)) + + +def minio_settings(environ: Mapping[str, str] | None = None) -> dict[str, Any]: + """Endpoint, keys and TLS from ``.env``/the environment; a missing key is named, a value never is.""" + if environ is None: + from ..config import load_dotenv + + load_dotenv() + environ = os.environ + missing = [k for k in MINIO_KEYS if not environ.get(k)] + if missing: + raise MemoryStoreError(f"OUSAST_MEMORY=minio://... needs {', '.join(missing)} in .env or the environment") + secure = (environ.get("MINIO_SECURE") or "true").strip().lower() not in ("0", "false", "no", "off") + return { + "endpoint": environ["MINIO_ENDPOINT"], "access_key": environ["MINIO_ACCESS_KEY"], + "secret_key": environ["MINIO_SECRET_KEY"], "secure": secure, + } # fmt: skip + + +def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None) -> MemoryStore: + """The store ``spec`` (else ``OUSAST_MEMORY``, else ``file:///memory``) names.""" + from .reconciler import results_root + + text = spec or (environ if environ is not None else os.environ).get("OUSAST_MEMORY") or "" + if not text: + return FileStore(results_root() / "memory") + parts = urlsplit(text) + if parts.scheme == "file" and parts.path: + return FileStore(Path(parts.path)) + if parts.scheme == "minio" and parts.netloc: + settings = minio_settings(environ) + return MinioStore(SdkClient(bucket=parts.netloc, **settings), parts.netloc, parts.path) + raise MemoryStoreError(f"OUSAST_MEMORY must be file:/// or minio://[/], got {text!r}") + + +# --- ingest and seed ----------------------------------------------------------------------------------------------- + +REMEMBER_OUTPUT = "memory.jsonl" + + +def read_memory(path: Path) -> list[dict[str, Any]]: + """A delivered ``memory.jsonl``: every row valid, else the error names the file and line.""" + rows: list[dict[str, Any]] = [] + for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if raw.strip(): + try: + rows.append(validate_row(json.loads(raw), f"{path}:{number}")) + except ValueError as exc: + raise MemoryStoreError(f"{path}:{number}: {exc}") from exc + return rows + + +def ingest(run_dir: Path, store: MemoryStore | None = None) -> list[IngestResult]: + """Every delivered ``/memory.jsonl`` of ``run_dir`` (and the ``facts.json`` beside it) into the store. + A run without remember outputs is a no-op that says so; a malformed file fails before anything is written.""" + store = store or open_store() + delivered = sorted(p for p in Path(run_dir).glob(f"*/{REMEMBER_OUTPUT}") if p.is_file()) + if not delivered: + log.info("memory: %s has no remember outputs; nothing to ingest", run_dir) + return [] + parsed = [(path, read_memory(path)) for path in delivered] + results: list[IngestResult] = [] + for path, rows in parsed: + facts = path.parent / "facts.json" + results.append(store.ingest_rows(Path(run_dir).name, path.parent.name, rows, facts.read_bytes() if facts.is_file() else None)) + return results + + +def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: Path | None = None) -> list[str]: + """Reuse stored facts for a Run's tasks whose ``openultrasast.io/memory-key`` matches (Req 6.1): each such task + without a directory or a state gets ``facts.json`` and a ``done`` summary and is marked done under the run + lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes.""" + from . import reconciler + + run_spec, manifests = reconciler.load_run(Path(run_manifest)) + store = store or open_store() + base = reconciler.run_dir(run_spec.metadata.name, results_root) + state = (reconciler._read_json(base / "state.json") or {}).get("tasks") or {} + seeded: list[str] = [] + for entry in run_spec.tasks: + text = manifests.tasks[entry.task].metadata.annotations.get(MEMORY_KEY_ANNOTATION) + if not text or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending": + continue + key = parse_memory_key(text) + where = {"candidates_digest": key["candidates"], "image": key["image"]} + for record in sorted(store.rows(key["repo"], key["pin"], "facts", where), key=lambda r: str(r.row["run"]), reverse=True): + data = store.get_facts(str(record.row["sha256"])) + if data is None: + continue + reused = {"sha256": record.row["sha256"], "from_run": record.row["run"], "image": key["image"]} + (base / entry.name).mkdir(parents=True) + (base / entry.name / "facts.json").write_bytes(data) + summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None} + (base / entry.name / "summary.json").write_text(json.dumps({**summary, "reused": reused}, indent=2, sort_keys=True) + "\n") + reconciler.mark_done(run_spec.metadata.name, entry.name, reused=reused, results_root=results_root) + seeded.append(entry.name) + break + return seeded + + +def counts(results: Iterable[IngestResult]) -> dict[str, int]: + total: dict[str, int] = {} + for result in results: + for kind, number in result.kinds.items(): + total[kind] = total.get(kind, 0) + number + return dict(sorted(total.items())) + + +__all__ = [ + "KINDS", + "MEMORY_KEY_ANNOTATION", + "POPULATION_ANNOTATION", + "SPLIT_ANNOTATION", + "FileStore", + "IngestResult", + "MemoryStore", + "MemoryStoreError", + "MinioStore", + "ObjectClient", + "Record", + "SdkClient", + "counts", + "image_digest", + "ingest", + "memory_key", + "open_store", + "parse_memory_key", + "repo_key", + "row_id", + "seed", + "validate_row", + "where_sql", +] diff --git a/src/openultrasast/plane/reconciler.py b/src/openultrasast/plane/reconciler.py index b94b2c3..de7ad92 100644 --- a/src/openultrasast/plane/reconciler.py +++ b/src/openultrasast/plane/reconciler.py @@ -257,6 +257,18 @@ def _acquire_lock(base: Path) -> Path: return lock +def mark_done(run_name: str, task: str, *, reused: Mapping[str, Any], results_root: Path | None = None) -> None: + """Set ``task`` done without running it (its outputs came from the memory store), under the run lock.""" + base = run_dir(run_name, results_root) + base.mkdir(parents=True, exist_ok=True) + lock = _acquire_lock(base) + try: + state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_name, "started": _now(), "tasks": {}}) + state.set(task, status="done", finished=_now(), usd=None, model=None, reused=dict(reused)) + finally: + lock.unlink(missing_ok=True) + + def _await(ax: Ax, name: str, delivered: Callable[[], bool], start: Callable[[], str | None]) -> str | None: """Resume ``name``, ``start`` it once Running (again after a re-resume), await delivery: None, else why it failed. ``Failed`` is final only once a resume went through (ax sets it when a resume call fails); until then a @@ -462,4 +474,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No return "\n".join(lines) -__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "outcome_of", "render_task", "run", "run_dir", "status"] +__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"] diff --git a/src/openultrasast/plane/tasks/remember.py b/src/openultrasast/plane/tasks/remember.py new file mode 100644 index 0000000..18b5193 --- /dev/null +++ b/src/openultrasast/plane/tasks/remember.py @@ -0,0 +1,246 @@ +"""`remember` (harnessx-removal Requirement 6.1, design section 4): a case's delivered artifacts as memory rows. + +No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the +verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present, +`alerts.jsonl`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`, `unit_cost` and `alert` (the +fifth kind, `proposal_outcome`, comes from the loop) -- and passes `facts.json` through. Task code interprets the +artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4). + +Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`, +`population`, `split` and `image` (the runner image digest the case's tasks ran on). + +Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS`, +`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes' +`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound +Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from +`OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and +`openultrasast.io/split` annotations). Exit 0 done, 2 failed. + +On the host, :func:`remember_run` does the same over a recorded run directory (``ousast plane remember ``): +per case without a delivered `memory.jsonl` it reads the artifacts and the rendered `task.yaml` beside them and +ingests the rows directly, writing nothing into the run directory. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import traceback +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from ..memory import REMEMBER_OUTPUT, IngestResult, MemoryStore, image_digest, ingest, repo_key, row_id, validate_row +from .repo_facts import _digest + +PASSES = ("a", "b", "c") +UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class Context: + run: str + task: str + repo: str + pin: str + image: str + population: str = UNKNOWN + split: str = UNKNOWN + + def row(self, kind: str, subject: str, **fields: Any) -> dict[str, Any]: + base = { + "id": row_id(kind, self.run, self.task, subject), "kind": kind, "repo": repo_key(self.repo), "pin": self.pin, + "run": self.run, "task": self.task, "population": self.population, "split": self.split, "image": self.image, + } # fmt: skip + return {**base, **fields} + + +def candidates_digest(facts: Mapping[str, Any]) -> str: + """`repo_facts._digest` of the candidate names `facts.json` was computed for (its `callers` keys).""" + names = sorted({str(key).partition("::")[2] for key in (facts.get("callers") or {})}) + return _digest(names) + + +def _final(row: Mapping[str, Any], disputed: set[str]) -> str: + if row.get("agreed"): + return "agreed" + return "disputed" if str(row["candidate"]) in disputed else "rejected" + + +def rows_for( + ctx: Context, + *, + facts: bytes | None, + passes: Mapping[str, Sequence[Mapping[str, Any]]], + models: Mapping[str, str | None], + agreed: Mapping[str, Any] | None, + alerts: Sequence[Mapping[str, Any]] = (), +) -> list[dict[str, Any]]: + """The memory rows of one case: facts, verdicts, unit costs, alerts.""" + rows: list[dict[str, Any]] = [] + if facts is not None: + loaded = json.loads(facts) + sha = hashlib.sha256(facts).hexdigest() + counts = loaded.get("counts") or {} + files = int(counts.get("files", len(loaded.get("files") or {}))) + rows.append(ctx.row("facts", sha, sha256=sha, candidates_digest=candidates_digest(loaded), files=files)) + if agreed is not None: + disputed = {str(d["candidate"]) for d in agreed.get("disputed") or []} + for cand in agreed.get("candidates") or []: + finding = cand.get("finding") or {} + fields = { + "candidate": cand["candidate"], "family": finding.get("family") or agreed.get("family"), "site": cand.get("site"), + "passes": {p: cand.get(p) for p in PASSES}, "final": _final(cand, disputed), "tiebreak": cand.get("c") is not None, + "site_match": cand.get("site_match"), "usd": cand.get("usd"), "turns": cand.get("turns"), + } # fmt: skip + rows.append(ctx.row("verdict", str(cand["candidate"]), **fields)) + for label in PASSES: + for unit in passes.get(label) or []: + usage = dict(unit.get("usage") or {}) + fields = { + "pass": label, "path": unit.get("path"), "usd": unit.get("usd"), "calls": usage.get("calls"), "usage": usage, + "turns": unit.get("turns"), "model": models.get(label), + } # fmt: skip + rows.append(ctx.row("unit_cost", f"{label}:{unit.get('path')}", **fields)) + for alert in alerts: + fields = {k: alert.get(k) for k in ("rule_id", "rule_status", "path", "line", "function", "pin_role")} + rows.append(ctx.row("alert", f"{fields['rule_id']}:{fields['path']}:{fields['line']}", **fields)) + return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows] + + +def _jsonl(path: Path | None) -> list[dict[str, Any]]: + if path is None or not path.is_file(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _json(path: Path | None) -> Any: + return json.loads(path.read_text(encoding="utf-8")) if path is not None and path.is_file() else None + + +def _model(summary: object) -> str | None: + return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None + + +def context_from( + task_doc: Mapping[str, Any], workspaces: Sequence[Mapping[str, Any]], *, run: str, task: str, population: str, split: str +) -> Context: + """Repository, pin and image from a rendered Task and its Workspaces: the pin named in `OUSAST_GIT_PINS`, the + repository of that Workspace's git entry, the Task's image.""" + spec = task_doc.get("spec") or {} + env = {str(e.get("name")): str(e.get("value")) for e in spec.get("env") or []} + pins: dict[str, str] = json.loads(env.get("OUSAST_GIT_PINS") or "{}") + if len(pins) != 1: + raise ValueError(f"{task}: expected one pinned repository in OUSAST_GIT_PINS, got {sorted(pins)}") + ((ref, pin),) = pins.items() + ws_name, _, git_name = ref.partition("/") + repos = [ + str(g["repo"]) + for w in workspaces if (w.get("metadata") or {}).get("name") == ws_name + for g in (w.get("spec") or {}).get("git") or [] if g.get("name") == git_name + ] # fmt: skip + if not repos: + raise ValueError(f"{task}: no bound Workspace git entry {ref}") + image = str(spec.get("image") or "") + if not image: + raise ValueError(f"{task}: the Task names no image") + return Context(run, task, repos[0], pin, image_digest(image), population or UNKNOWN, split or UNKNOWN) + + +def _rendered(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + docs = [d for d in yaml.safe_load_all(path.read_text(encoding="utf-8")) if isinstance(d, dict)] + tasks = [d for d in docs if d.get("kind") == "Task"] + if not tasks: + raise ValueError(f"{path}: no rendered Task") + return tasks[-1], [d for d in docs if d.get("kind") == "Workspace"] + + +def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[list[dict[str, Any]], bytes | None]: + """The rows of one case of a recorded run directory, from its artifacts and the rendered `task.yaml`.""" + facts_dir = base / f"{case}-facts" + candidates = [base / f"{case}-{step}" / "task.yaml" for step in ("facts", "va", "vb", "agree")] + rendered = next((path for path in candidates if path.is_file()), None) + if rendered is None: + raise ValueError(f"{base}: case {case} has no rendered task.yaml") + task_doc, workspaces = _rendered(rendered) + ctx = context_from(task_doc, workspaces, run=base.name, task=f"{case}-remember", population=population, split=split) + reused = (_json(facts_dir / "summary.json") or {}).get("reused") + if isinstance(reused, dict) and reused.get("image"): + ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, image_digest(str(reused["image"])), ctx.population, ctx.split) + facts = (facts_dir / "facts.json").read_bytes() if (facts_dir / "facts.json").is_file() else None + final = base / f"{case}-final" / "agreed.json" + agreed = _json(final if final.is_file() else base / f"{case}-agree" / "agreed.json") + passes = {p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES} + models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES} + alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl") + return rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts), facts + + +def remember_run(run_dir: Path, store: MemoryStore, *, population: str = UNKNOWN, split: str = UNKNOWN) -> list[IngestResult]: + """Ingest a run: delivered `memory.jsonl` files as they are, and every other case (a `-facts` task) + computed here from its artifacts.""" + base = Path(run_dir) + if not base.is_dir(): + raise ValueError(f"no run directory at {base}") + results = ingest(base, store) + delivered = {r.task for r in results} + for facts_dir in sorted(base.glob("*-facts")): + case = facts_dir.name.removesuffix("-facts") + if f"{case}-remember" in delivered or (base / f"{case}-remember" / REMEMBER_OUTPUT).is_file(): + continue + rows, facts = case_rows(base, case, population=population, split=split) + results.append(store.ingest_rows(base.name, f"{case}-remember", rows, facts)) + return results + + +def _path(env: Mapping[str, str], name: str) -> Path | None: + return Path(env[name]) if env.get(name) else None + + +def _units(path: Path | None) -> list[dict[str, Any]]: + return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path) + + +def main(environ: Mapping[str, str] | None = None) -> int: + env = environ if environ is not None else os.environ + output_dir = _path(env, "OUSAST_OUTPUT_DIR") + summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None} + try: + if output_dir is None: + raise ValueError("OUSAST_OUTPUT_DIR is not set") + output_dir.mkdir(parents=True, exist_ok=True) + task_doc = yaml.safe_load(env.get("AX_TASK_YAML") or "{}") or {} + task_doc.setdefault("spec", {})["env"] = [{"name": "OUSAST_GIT_PINS", "value": env.get("OUSAST_GIT_PINS") or "{}"}] + workspaces = [d for d in yaml.safe_load_all(env.get("AX_WORKSPACES_YAML") or "") if isinstance(d, dict)] + ctx = context_from( + task_doc, workspaces, run=env.get("OUSAST_RUN") or UNKNOWN, task=env.get("OUSAST_TASK") or "remember", + population=env.get("OUSAST_POPULATION") or UNKNOWN, split=env.get("OUSAST_SPLIT") or UNKNOWN, + ) # fmt: skip + facts_path = _path(env, "OUSAST_INPUT_FACTS") + facts = facts_path.read_bytes() if facts_path is not None and facts_path.is_file() else None + passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES} + models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES} + agreed = _json(_path(env, "OUSAST_INPUT_AGREED")) + rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS"))) + (output_dir / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8") + if facts is not None: + (output_dir / "facts.json").write_bytes(facts) + summary.update(status="done", units_done=1, rows=len(rows)) + except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback + summary["reason"] = traceback.format_exc()[-2000:] + if output_dir is not None: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({k: summary.get(k) for k in ("status", "rows")}), file=sys.stderr) + return 0 if summary["status"] == "done" else 2 + + +__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "main", "remember_run", "rows_for"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/agreed.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/agreed.json new file mode 100644 index 0000000..237321b --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/agreed.json @@ -0,0 +1,59 @@ +{ + "family": "untrusted_destination", + "candidates": [ + { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "a": true, + "b": true, + "agreed": true, + "disputed": false, + "site_match": true, + "usd": 0.022175, + "turns": 10, + "finding": { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "reported_at": "lmdeploy/vl/media/connection.py:68", + "family": "untrusted_destination", + "title": "SSRF guard bypass via HTTP redirect in _load_http_url", + "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c" + } + } + ], + "agreed": [ + { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "reported_at": "lmdeploy/vl/media/connection.py:68", + "family": "untrusted_destination", + "title": "SSRF guard bypass via HTTP redirect in _load_http_url", + "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c", + "passes": 2 + } + ], + "disputed": [], + "metrics": { + "candidates": 1, + "candidates_before_triage": 1, + "asked_in_both": 1, + "flagged_a": 1, + "flagged_b": 1, + "agreed": 1, + "disputed": 0, + "hunts": 2, + "declared_sites": 1, + "declared_sites_in_set": 1, + "declared_sites_matched": 1, + "declared_sites_matched_agreed": 1, + "site_matching": "assessed", + "usd_total": 0.022175, + "cost_per_candidate": 0.022175, + "recorded_triage_usd": 0.000768, + "recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded", + "usd_total_with_recorded_triage": 0.022943, + "cost_per_candidate_with_recorded_triage": 0.022943, + "turns_per_hunt": 5.0, + "calls_per_hunt": 7.0 + } +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/disputed.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/disputed.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/disputed.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/summary.json new file mode 100644 index 0000000..6008d2f --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-agree/summary.json @@ -0,0 +1,38 @@ +{ + "status": "done", + "units_done": 1, + "units_total": 1, + "usd": 0, + "calls": 0, + "usage": { + "prompt_tokens": 0, + "prompt_cache_hit_tokens": 0, + "completion_tokens": 0 + }, + "priced": true, + "model": null, + "pass": null, + "metrics": { + "candidates": 1, + "candidates_before_triage": 1, + "asked_in_both": 1, + "flagged_a": 1, + "flagged_b": 1, + "agreed": 1, + "disputed": 0, + "hunts": 2, + "declared_sites": 1, + "declared_sites_in_set": 1, + "declared_sites_matched": 1, + "declared_sites_matched_agreed": 1, + "site_matching": "assessed", + "usd_total": 0.022175, + "cost_per_candidate": 0.022175, + "recorded_triage_usd": 0.000768, + "recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded", + "usd_total_with_recorded_triage": 0.022943, + "cost_per_candidate_with_recorded_triage": 0.022943, + "turns_per_hunt": 5.0, + "calls_per_hunt": 7.0 + } +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/facts.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/facts.json new file mode 100644 index 0000000..1bb730f --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/facts.json @@ -0,0 +1,22 @@ +{ + "callers": { + "lmdeploy/vl/media/connection.py::_load_http_url": [] + }, + "counts": { + "callers": 0, + "candidates": 1, + "files": 1, + "functions": 4809 + }, + "files": { + "lmdeploy/vl/media/connection.py": { + "functions": [ + "_is_safe_url", + "_load_http_url", + "_load_data_url", + "_load_file_url", + "load_from_url" + ] + } + } +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/summary.json new file mode 100644 index 0000000..d2567cf --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/summary.json @@ -0,0 +1,9 @@ +{ + "calls": 0, + "model": null, + "status": "done", + "units_done": 608, + "units_total": 608, + "usage": {}, + "usd": null +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/task.yaml b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/task.yaml new file mode 100644 index 0000000..1e904ac --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-facts/task.yaml @@ -0,0 +1,40 @@ +apiVersion: ax.io/v1alpha1 +kind: Workspace +metadata: + name: lmdeploy-media-url-ssrf-vulnerable + atespace: default +spec: + git: + - name: repo + repo: https://example.com/reserved/lmdeploy + dir: repo + depth: 1 +--- +apiVersion: ax.io/v1alpha1 +kind: Task +metadata: + name: validation-46-lmdeploy-media-url-ssrf-facts + atespace: default +spec: + command: + - repo-facts + env: + - name: OUSAST_RUN + value: validation-46 + - name: OUSAST_TASK + value: lmdeploy-media-url-ssrf-facts + - name: OUSAST_GIT_PINS + value: '{"lmdeploy-media-url-ssrf-vulnerable/repo": "71648d239c978452720a6d6692fe914efc8a7b94"}' + image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: '1' + memory: 1280Mi + workspaces: + - name: lmdeploy-media-url-ssrf-vulnerable + path: /workspace/case + - name: lmdeploy-media-url-ssrf-inputs + path: /workspace/inputs diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/agreed.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/agreed.json new file mode 100644 index 0000000..291c565 --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/agreed.json @@ -0,0 +1,66 @@ +{ + "family": "untrusted_destination", + "candidates": [ + { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "a": true, + "b": true, + "c": null, + "agreed": true, + "disputed": false, + "site_match": true, + "usd": 0.022175, + "turns": 10, + "finding": { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "reported_at": "lmdeploy/vl/media/connection.py:68", + "family": "untrusted_destination", + "title": "SSRF guard bypass via HTTP redirect in _load_http_url", + "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c" + } + } + ], + "agreed": [ + { + "candidate": "lmdeploy/vl/media/connection.py::_load_http_url", + "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", + "reported_at": "lmdeploy/vl/media/connection.py:68", + "family": "untrusted_destination", + "title": "SSRF guard bypass via HTTP redirect in _load_http_url", + "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c", + "passes": 2 + } + ], + "disputed": [], + "metrics": { + "candidates": 1, + "candidates_before_triage": 1, + "asked_in_both": 1, + "flagged_a": 1, + "flagged_b": 1, + "agreed": 1, + "disputed": 0, + "hunts": 2, + "declared_sites": 1, + "declared_sites_in_set": 1, + "declared_sites_matched": 1, + "declared_sites_matched_agreed": 1, + "site_matching": "assessed", + "usd_total": 0.022175, + "cost_per_candidate": 0.022175, + "recorded_triage_usd": 0.000768, + "recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded", + "usd_total_with_recorded_triage": 0.022943, + "cost_per_candidate_with_recorded_triage": 0.022943, + "turns_per_hunt": 5.0, + "calls_per_hunt": 7.0, + "tiebreak": { + "asked": 0, + "flipped_to_agreed": 0, + "flipped_to_rejected": 0, + "rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)" + } + } +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/disputed.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/disputed.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/disputed.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/summary.json new file mode 100644 index 0000000..3e9ed7f --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-final/summary.json @@ -0,0 +1,44 @@ +{ + "status": "done", + "units_done": 1, + "units_total": 1, + "usd": 0, + "calls": 0, + "usage": { + "prompt_tokens": 0, + "prompt_cache_hit_tokens": 0, + "completion_tokens": 0 + }, + "priced": true, + "model": null, + "pass": null, + "metrics": { + "candidates": 1, + "candidates_before_triage": 1, + "asked_in_both": 1, + "flagged_a": 1, + "flagged_b": 1, + "agreed": 1, + "disputed": 0, + "hunts": 2, + "declared_sites": 1, + "declared_sites_in_set": 1, + "declared_sites_matched": 1, + "declared_sites_matched_agreed": 1, + "site_matching": "assessed", + "usd_total": 0.022175, + "cost_per_candidate": 0.022175, + "recorded_triage_usd": 0.000768, + "recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded", + "usd_total_with_recorded_triage": 0.022943, + "cost_per_candidate_with_recorded_triage": 0.022943, + "turns_per_hunt": 5.0, + "calls_per_hunt": 7.0, + "tiebreak": { + "asked": 0, + "flipped_to_agreed": 0, + "flipped_to_rejected": 0, + "rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)" + } + } +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/summary.json new file mode 100644 index 0000000..9e9010b --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/summary.json @@ -0,0 +1,16 @@ +{ + "status": "done", + "units_done": 1, + "units_total": 1, + "usd": 0.012376, + "calls": 7, + "usage": { + "prompt_tokens": 68714, + "prompt_cache_hit_tokens": 48512, + "completion_tokens": 2127 + }, + "priced": true, + "model": "deepseek-flash", + "pass": "a", + "family": "untrusted_destination" +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/units.jsonl b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/units.jsonl new file mode 100644 index 0000000..e008b99 --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-va/units.jsonl @@ -0,0 +1 @@ +{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "a", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF guard bypass via HTTP redirect in _load_http_url", "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"}], "turns": 5, "usage": {"prompt_tokens": 68714, "prompt_cache_hit_tokens": 48512, "completion_tokens": 2127, "calls": 7}, "usd": 0.01237569} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/summary.json new file mode 100644 index 0000000..9c7358f --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/summary.json @@ -0,0 +1,16 @@ +{ + "status": "done", + "units_done": 1, + "units_total": 1, + "usd": 0.009799, + "calls": 7, + "usage": { + "prompt_tokens": 61163, + "prompt_cache_hit_tokens": 46464, + "completion_tokens": 2031 + }, + "priced": true, + "model": "deepseek-flash", + "pass": "b", + "family": "untrusted_destination" +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/units.jsonl b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/units.jsonl new file mode 100644 index 0000000..172669d --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vb/units.jsonl @@ -0,0 +1 @@ +{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "b", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF via unvalidated HTTP redirects in media URL fetch", "witness": "Source: an HTTP client submits a chat request whose multimodal content contains an attacker-controlled URL, e.g. {'type'"}], "turns": 5, "usage": {"prompt_tokens": 61163, "prompt_cache_hit_tokens": 46464, "completion_tokens": 2031, "calls": 7}, "usd": 0.00979898} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/summary.json b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/summary.json new file mode 100644 index 0000000..a7f4c02 --- /dev/null +++ b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/summary.json @@ -0,0 +1,17 @@ +{ + "status": "done", + "units_done": 0, + "units_total": 0, + "usd": 0.0, + "calls": 0, + "usage": { + "prompt_tokens": 0, + "prompt_cache_hit_tokens": 0, + "completion_tokens": 0 + }, + "priced": true, + "model": "deepseek-flash", + "pass": "c", + "family": "untrusted_destination", + "only": 0 +} diff --git a/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/units.jsonl b/tests/fixtures/plane-remember/validation-46/lmdeploy-media-url-ssrf-vc/units.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_plane_increment.py b/tests/test_plane_increment.py index 96df2a6..fec33f6 100644 --- a/tests/test_plane_increment.py +++ b/tests/test_plane_increment.py @@ -19,6 +19,8 @@ from openultrasast.plane import reconciler from openultrasast.plane.budget import prices_from from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates from openultrasast.plane.manifests import load_manifests +from openultrasast.plane.memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, parse_memory_key +from openultrasast.plane.tasks.repo_facts import _digest PLANE = Path("plane") RUN = PLANE / "runs" / "validation-46.yaml" @@ -160,7 +162,8 @@ def test_committed_manifests_match_the_generator() -> None: POPULATION, REPOS, ) - for relative, text in render(cases, templates, "validation-46", _committed_command()).items(): + rendered = render(cases, templates, "validation-46", _committed_command(), population=POPULATION.stem, split="validation") + for relative, text in rendered.items(): assert (PLANE / relative).read_text() == text, relative @@ -216,6 +219,12 @@ def test_generator_is_deterministic_and_follows_the_reference(tmp_path: Path) -> assert inputs["triage.json"]["candidates"] == [["a.py", "helper", 20], ["a.py", "run", 9]] assert run.task("alpha-va").budget is not None and run.task("alpha-va").budget.usd == 0.38 # 1.5 * 0.5 / 2, cent up assert yaml.safe_load(first[0].read_text())["metadata"]["annotations"] == {"openultrasast.io/git-commits": f"repo={PIN}"} + # memory (harnessx-removal §4): the facts Task names its reuse key, the Run its population and split + key = parse_memory_key(manifests.tasks["repo-facts-alpha"].metadata.annotations[MEMORY_KEY_ANNOTATION]) + image = manifests.tasks["repo-facts-alpha"].image or "" + assert key == {"repo": "example.com/alpha", "pin": PIN, "candidates": _digest(["run"]), "image": image.rpartition("@")[2]} + assert all(MEMORY_KEY_ANNOTATION not in t.metadata.annotations for n, t in manifests.tasks.items() if not n.startswith("repo-facts-")) + assert dict(run.metadata.annotations) == {POPULATION_ANNOTATION: "population", SPLIT_ANNOTATION: "set"} def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path) -> None: diff --git a/tests/test_plane_memory.py b/tests/test_plane_memory.py new file mode 100644 index 0000000..e16d3d6 --- /dev/null +++ b/tests/test_plane_memory.py @@ -0,0 +1,383 @@ +"""The plane's memory store (harnessx-removal Req 6.1/6.4, design §4): one contract over every backend, the S3 +Select probe and its fallback on a fake object client, backend selection, fact reuse by ``seed``, disk refusal.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import shutil +import uuid +from collections.abc import Iterator +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest + +from openultrasast.plane import memory, reconciler +from openultrasast.plane.memory import ( + MINIO_KEYS, + FileStore, + MemoryStore, + MemoryStoreError, + MinioStore, + SdkClient, + memory_key, + minio_settings, + open_store, + row_id, + seed, + where_sql, +) +from openultrasast.plane.tasks.repo_facts import _digest + +PIN = "0123456789abcdef0123456789abcdef01234567" +OTHER_PIN = "fedcba9876543210fedcba9876543210fedcba98" +IMAGE = "sha256:" + "a" * 64 +FACTS = json.dumps({"callers": {"a.py::run": []}, "counts": {"files": 1}, "files": {"a.py": {"functions": ["run"]}}}).encode() +MINIO_SKIP = ( + "MinIO contract tests need OUSAST_MEMORY_TEST_MINIO=1 plus MINIO_ENDPOINT, MINIO_ACCESS_KEY and MINIO_SECRET_KEY " + "exported (optional: MINIO_SECURE, OUSAST_MEMORY_TEST_BUCKET, default ousast-memory-test) and the minio extra installed" +) + + +def row(kind: str, subject: str, *, pin: str = PIN, run: str = "r1", repo: str = "https://github.com/o/r", **fields: Any) -> dict[str, Any]: + task = "c-remember" + base = {"id": row_id(kind, run, task, subject), "kind": kind, "repo": memory.repo_key(repo), "pin": pin, "run": run, "task": task} + return {**base, "population": "population-v2", "split": "validation", "image": IMAGE, **fields} + + +def verdict(candidate: str, final: str, **fields: Any) -> dict[str, Any]: + return row("verdict", candidate, candidate=candidate, family="injection", final=final, tiebreak=False, **fields) + + +class FakeObjects: + """An in-memory :class:`memory.ObjectClient`: versions per write, tags per object; ``select`` works, raises or + answers wrongly, evaluating the equality clauses :func:`where_sql` writes.""" + + def __init__(self, select: str = "works") -> None: + self.mode, self.objects, self.versions = select, {}, {} + self.selects: list[str] = [] + self.gets: list[str] = [] + self.calls: list[tuple[Any, ...]] = [] + + def get(self, key: str) -> tuple[bytes, str | None] | None: + self.gets.append(key) + found = self.objects.get(key) + return None if found is None else (found[0], f"v{self.versions[key]}") + + def put(self, key: str, data: bytes, labels: dict[str, str]) -> None: + self.versions[key] = self.versions.get(key, 0) + 1 + self.objects[key] = (data, dict(labels)) + + def delete(self, key: str) -> None: + self.objects.pop(key, None) + + def keys(self, prefix: str) -> list[str]: + return sorted(k for k in self.objects if k.startswith(prefix)) + + def tags(self, key: str) -> dict[str, str]: + return dict(self.objects[key][1]) + + def version(self, key: str) -> str | None: + return f"v{self.versions[key]}" if key in self.objects else None + + def select(self, key: str, expression: str) -> bytes: + self.selects.append(expression) + if self.mode == "raises": + raise RuntimeError("NotImplemented: SelectObjectContent is not supported") + if self.mode == "wrong": + return b"" + clauses = re.findall(r"s\.\"(\w+)\" = ('(?:[^']|'')*'|true|false|-?[\d.]+)", expression) + want = {n: v[1:-1].replace("''", "'") if v.startswith("'") else json.loads(v) for n, v in clauses} + rows = [json.loads(line) for line in self.objects[key][0].splitlines() if line.strip()] + return b"".join(json.dumps(r).encode() + b"\n" for r in rows if all(r.get(k) == v for k, v in want.items())) + + def presign(self, method: str, key: str, expires: timedelta) -> str: + return f"https://minio.test/{key}?method={method}&X-Amz-Expires={int(expires.total_seconds())}" + + def expire(self, prefix: str, days: int, rule_id: str) -> None: + self.calls.append(("expire", prefix, days, rule_id)) + + def enable_versioning(self) -> None: + self.calls.append(("versioning",)) + + +@pytest.fixture(autouse=True) +def fresh_probes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(memory, "_SELECT_PROBES", {}) + + +def _real_minio() -> Iterator[MemoryStore]: + if os.environ.get("OUSAST_MEMORY_TEST_MINIO") != "1" or any(not os.environ.get(k) for k in MINIO_KEYS): + pytest.skip(MINIO_SKIP) + pytest.importorskip("minio", reason=MINIO_SKIP) + bucket = os.environ.get("OUSAST_MEMORY_TEST_BUCKET") or "ousast-memory-test" + client = SdkClient(bucket=bucket, **minio_settings(os.environ)) + if not client.sdk.bucket_exists(bucket): + client.sdk.make_bucket(bucket) + store = MinioStore(client, bucket, f"contract-{uuid.uuid4().hex[:12]}") + client.enable_versioning() + yield store + for key in client.keys(store.prefix + "/"): + client.delete(key) + + +@pytest.fixture(params=["file", "fake-minio-select", "fake-minio-fallback", "minio"]) +def store(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[MemoryStore]: + if request.param == "file": + yield FileStore(tmp_path / "memory") + elif request.param == "minio": + yield from _real_minio() + else: + yield MinioStore(FakeObjects("works" if request.param.endswith("select") else "raises"), "bucket", "p") + + +# --- the contract -------------------------------------------------------------------------------------------------- + + +def test_ingest_is_idempotent_by_the_index(store: MemoryStore) -> None: + rows = [verdict("a.py::run", "agreed"), verdict("a.py::other", "rejected")] + first = store.ingest_rows("r1", "c-remember", rows, FACTS) + again = store.ingest_rows("r1", "c-remember", rows, FACTS) + assert (first.skipped, again.skipped, first.kinds) == (False, True, {"verdict": 2}) + assert [(e["run"], e["task"], e["rows"]) for e in store.index()] == [("r1", "c-remember", 2)] + assert len(store.rows(kind="verdict")) == 2 + + +def test_identical_facts_are_stored_once_and_read_back_by_content(store: MemoryStore) -> None: + sha = hashlib.sha256(FACTS).hexdigest() + store.ingest_rows("r1", "c-remember", [row("facts", sha, sha256=sha)], FACTS) + store.ingest_rows("r2", "c-remember", [row("facts", sha, run="r2", sha256=sha)], FACTS) + assert store._keys("facts/") == [f"facts/{sha}.json"] + assert store.get_facts(sha) == FACTS and store.get_facts("0" * 64) is None + + +def test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version(store: MemoryStore) -> None: + store.put_rows([ + verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), verdict("b.py::y", "agreed", pin=OTHER_PIN), + verdict("c.py::z", "agreed", repo="https://github.com/o/other.git"), row("unit_cost", "a:a.py", **{"pass": "a", "usd": 0.5}), + ]) # fmt: skip + assert len(store.rows()) == 5 + assert {r.row["candidate"] for r in store.rows("https://github.com/o/r", kind="verdict")} == {"a.py::run", "a.py::x", "b.py::y"} + assert [r.row["candidate"] for r in store.rows("github.com/o/r", PIN, "verdict", {"final": "disputed"})] == ["a.py::x"] + assert [r.row["kind"] for r in store.rows(where={"usd": 0.5})] == ["unit_cost"] + assert store.rows(where={"final": "no such"}) == [] + record = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0] + assert record.key == f"repos/github.com__o__r/{PIN}.jsonl" and record.version + store.put_row(verdict("a.py::new", "agreed")) + later = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0] + assert later.row == record.row and later.version != record.version, "a rewrite is a new version: provenance pins the old" + + +def test_a_repeated_id_replaces_the_stored_row(store: MemoryStore) -> None: + store.put_row(verdict("a.py::run", "disputed")) + store.put_row(verdict("a.py::run", "agreed")) + assert [r.row["final"] for r in store.rows(kind="verdict")] == ["agreed"] + + +def test_a_malformed_row_fails_naming_it_and_writes_nothing(store: MemoryStore) -> None: + bad = {**verdict("a.py::run", "agreed"), "pin": "main"} + with pytest.raises(MemoryStoreError, match=r"r1/c-remember row 2: pin 'main'"): + store.ingest_rows("r1", "c-remember", [verdict("a.py::ok", "agreed"), bad]) + with pytest.raises(MemoryStoreError, match="unknown kind"): + store.put_row({**verdict("a.py::run", "agreed"), "kind": "vibes"}) + assert store.index() == [] and store.rows() == [] + + +def test_facts_that_no_longer_match_their_hash_are_dropped(store: MemoryStore, caplog: pytest.LogCaptureFixture) -> None: + sha = store.put_facts(FACTS) + store._put(f"facts/{sha}.json", b"tampered") + with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"): + assert store.get_facts(sha) is None + assert "no longer matches" in caplog.text and store._get(f"facts/{sha}.json") is None + + +def test_presigned_urls_name_the_object(store: MemoryStore) -> None: + for url in (store.presign_put("runs/r1/x/units.jsonl"), store.presign_get("runs/r1/x/units.jsonl")): + assert url.startswith(("file://", "http://", "https://")) and "runs/r1/x/units.jsonl" in url + + +# --- MinIO specifics on the fake client ---------------------------------------------------------------------------- + + +def _filled(client: FakeObjects) -> MinioStore: + store = MinioStore(client, "bucket") + store.put_rows([verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), row("unit_cost", "a:a.py", **{"pass": "a"})]) + return store + + +def test_select_is_pushed_down_when_the_probe_answers(caplog: pytest.LogCaptureFixture) -> None: + client = FakeObjects("works") + store = _filled(client) + with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"): + got = [r.row["candidate"] for r in store.rows(kind="verdict", where={"final": "disputed"})] + assert got == ["a.py::x"] and store.select_supported() and "locally" not in caplog.text + assert client.selects[0] == 'SELECT * FROM S3Object s WHERE s."probe" = 1' + assert client.selects[1:] == ["SELECT * FROM S3Object s WHERE s.\"final\" = 'disputed' AND s.\"kind\" = 'verdict'"] + + +@pytest.mark.parametrize("mode", ["raises", "wrong"]) +def test_without_select_the_same_rows_come_from_a_local_filter_logged_once( + mode: str, caplog: pytest.LogCaptureFixture, tmp_path: Path +) -> None: + client = FakeObjects(mode) + store, reference = _filled(client), FileStore(tmp_path / "m") + reference.put_rows([r.row for r in store.rows()]) + with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"): + for where in ({"final": "disputed"}, {"final": "agreed"}, {"pass": "a"}): + assert [r.row for r in store.rows(where=where)] == [r.row for r in reference.rows(where=where)] != [] + assert not store.select_supported() and len(client.selects) == 1, "one probe per process" + assert caplog.text.count("does not answer S3 Select") == 1 + + +def test_a_failing_select_on_one_object_falls_back_for_that_object(caplog: pytest.LogCaptureFixture) -> None: + client = FakeObjects("works") + store = _filled(client) + assert store.select_supported() + client.mode = "raises" + with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"): + assert [r.row["candidate"] for r in store.rows(where={"final": "agreed"})] == ["a.py::run"] + assert "filtering it locally" in caplog.text + + +def test_row_objects_carry_tags_and_a_kind_tag_skips_objects_without_that_kind() -> None: + client = FakeObjects("works") + store = MinioStore(client, "bucket") + store.put_rows([verdict("a.py::run", "agreed"), row("unit_cost", "a:b.py", pin=OTHER_PIN, **{"pass": "a"})]) + tags = client.tags(f"repos/github.com__o__r/{PIN}.jsonl") + assert tags == {"repo": "github.com/o/r", "pin": PIN, "kind": "verdict", "family": "injection", "run": "r1", + "population": "population-v2", "split": "validation"} # fmt: skip + client.gets.clear() + assert [r.row["kind"] for r in store.rows(kind="unit_cost")] == ["unit_cost"] + assert f"repos/github.com__o__r/{PIN}.jsonl" not in client.gets, "an object tagged without the kind is not read" + + +def test_configure_turns_versioning_on_and_expires_runs_only() -> None: + client = FakeObjects() + MinioStore(client, "bucket", "memory").configure(14) + assert client.calls == [("versioning",), ("expire", "memory/runs/", 14, "ousast-runs-expiry")] + with pytest.raises(ValueError): + MinioStore(client, "bucket").configure(0) + + +def test_where_sql_quotes_values_and_refuses_odd_fields() -> None: + assert where_sql({}) == "SELECT * FROM S3Object s" + assert where_sql({"a": "it's", "b": True, "c": None, "d": 2}) == ( + 'SELECT * FROM S3Object s WHERE s."a" = \'it\'\'s\' AND s."b" = true AND s."c" IS NULL AND s."d" = 2' + ) + with pytest.raises(ValueError): + where_sql({"a; drop": 1}) + + +# --- selection and settings ---------------------------------------------------------------------------------------- + + +def test_open_store_selects_the_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path)) + monkeypatch.delenv("OUSAST_MEMORY", raising=False) + default = open_store() + assert isinstance(default, FileStore) and default.root == tmp_path / "plane" / "memory" + chosen = open_store(environ={"OUSAST_MEMORY": f"file://{tmp_path}/m"}) + assert isinstance(chosen, FileStore) and chosen.root == tmp_path / "m" + with pytest.raises(MemoryStoreError, match="file:/// or minio://"): + open_store("s3://bucket") + + +def test_minio_settings_name_missing_keys_never_values() -> None: + with pytest.raises(MemoryStoreError, match="MINIO_ACCESS_KEY, MINIO_SECRET_KEY") as caught: + open_store(environ={"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "minio.example:9000"}) + assert "minio.example" not in str(caught.value) + env = {"MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "sk-value", "MINIO_SECURE": "false"} + assert minio_settings(env) == {"endpoint": "h:9000", "access_key": "ak", "secret_key": "sk-value", "secure": False} + assert minio_settings({**env, "MINIO_SECURE": ""})["secure"] is True + + +def test_minio_without_the_sdk_names_the_extra() -> None: + try: + import minio # noqa: F401 + except ImportError: + pass + else: + pytest.skip("the minio SDK is installed here") + env = {"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "secret-xyz"} + with pytest.raises(MemoryStoreError, match=r"install openultrasast\[minio\]") as caught: + open_store(environ=env) + assert "secret-xyz" not in str(caught.value) + + +# --- disk refusal (FileStore) -------------------------------------------------------------------------------------- + + +def test_file_store_refuses_below_one_gib_free_naming_its_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = tmp_path / "memory" + monkeypatch.setattr(memory.shutil, "disk_usage", lambda path: shutil._ntuple_diskusage(100 << 30, 99 << 30, 512 << 20)) # type: ignore[attr-defined] + with pytest.raises(MemoryStoreError, match=rf"refusing to write the memory store at {re.escape(str(root))}: 0.50 GiB free"): + FileStore(root).ingest_rows("r1", "c-remember", [verdict("a.py::run", "agreed")], FACTS) + assert not root.exists() + + +# --- fact reuse ---------------------------------------------------------------------------------------------------- + + +def _plane(root: Path, key: str, run: str = "r2") -> Path: + (root / "runs").mkdir(parents=True) + (root / "tasks").mkdir() + (root / "tasks" / "t.yaml").write_text( + "apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: repo-facts-c\n annotations:\n" + f" {memory.MEMORY_KEY_ANNOTATION}: '{key}'\nspec:\n image: reg/img@{IMAGE}\n command: [repo-facts]\n" + ) + (root / "runs" / f"{run}.yaml").write_text( + f"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata:\n name: {run}\nspec:\n tasks:\n" + " - name: c-facts\n task: repo-facts-c\n outputs: [facts.json]\n" + ) + return root / "runs" / f"{run}.yaml" + + +@pytest.fixture +def stored(tmp_path: Path) -> FileStore: + store = FileStore(tmp_path / "memory") + sha = hashlib.sha256(FACTS).hexdigest() + facts = row("facts", sha, sha256=sha, candidates_digest=_digest(["run"]), files=1) + store.ingest_rows("r1", "c-remember", [facts], FACTS) + return store + + +def test_seed_reuses_facts_for_an_unchanged_key_and_marks_the_task_done(stored: FileStore, tmp_path: Path) -> None: + manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}")) + results = tmp_path / "results" + assert seed(manifest, stored, results_root=results) == ["c-facts"] + base = results / "r2" + assert (base / "c-facts" / "facts.json").read_bytes() == FACTS + summary = json.loads((base / "c-facts" / "summary.json").read_text()) + reused = {"sha256": hashlib.sha256(FACTS).hexdigest(), "from_run": "r1", "image": IMAGE} + assert reconciler.outcome_of(summary) == ("done", "") and summary["reused"] == reused + state = json.loads((base / "state.json").read_text()) + assert state["tasks"]["c-facts"]["status"] == "done" and state["tasks"]["c-facts"]["reused"] == reused + assert not (base / "lock").exists() + assert seed(manifest, stored, results_root=results) == [], "a seeded task is never seeded again" + + +@pytest.mark.parametrize( + "change", [{"pin": OTHER_PIN}, {"candidates": _digest(["run", "helper"])}, {"image": "reg/img@sha256:" + "b" * 64}] +) +def test_seed_recomputes_on_a_changed_pin_candidate_set_or_image(stored: FileStore, tmp_path: Path, change: dict[str, str]) -> None: + key = {"repo": "https://github.com/o/r", "pin": PIN, "candidates": _digest(["run"]), "image": f"reg/img@{IMAGE}", **change} + manifest = _plane(tmp_path / "plane", memory_key(**key)) + assert seed(manifest, stored, results_root=tmp_path / "results") == [] + assert not (tmp_path / "results" / "r2").exists() + + +def test_seed_never_overwrites_a_task_directory_and_drops_tampered_facts(stored: FileStore, tmp_path: Path) -> None: + manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}")) + results = tmp_path / "results" + (results / "r2" / "c-facts").mkdir(parents=True) + assert seed(manifest, stored, results_root=results) == [] + (results / "r2" / "c-facts").rmdir() + sha = hashlib.sha256(FACTS).hexdigest() + (stored.root / "facts" / f"{sha}.json").write_bytes(b"changed underneath") + assert seed(manifest, stored, results_root=results) == [] + assert not (stored.root / "facts" / f"{sha}.json").exists() diff --git a/tests/test_plane_reconciler.py b/tests/test_plane_reconciler.py index 70a06cf..83497ea 100644 --- a/tests/test_plane_reconciler.py +++ b/tests/test_plane_reconciler.py @@ -756,6 +756,22 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None: assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs] +def test_mark_done_sets_the_state_under_the_run_lock(tmp_path: Path) -> None: + import os + + base = tmp_path / "r" + base.mkdir() + (base / "lock").write_text(json.dumps(os.getppid())) # a live process that is not this one holds the run + with pytest.raises(RuntimeError, match="run is held by pid"): + reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path) + assert not (base / "state.json").exists() + (base / "lock").unlink() + reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path) + task = json.loads((base / "state.json").read_text())["tasks"]["c-facts"] + assert (task["status"], task["reused"]) == ("done", {"sha256": "ab", "from_run": "r0"}) and task["finished"] + assert not (base / "lock").exists(), "the lock is released" + + def test_reconciler_is_under_500_lines_and_holds_no_pipeline_logic() -> None: text = RECONCILER.read_text(encoding="utf-8") assert len(text.splitlines()) < 500, "Req 3.4: the reconciler stays under 500 lines" diff --git a/tests/test_plane_remember.py b/tests/test_plane_remember.py new file mode 100644 index 0000000..4af5b94 --- /dev/null +++ b/tests/test_plane_remember.py @@ -0,0 +1,171 @@ +"""`remember` (harnessx-removal §4) on a recorded Run: a few KB of validation-46's lmdeploy case under +``tests/fixtures/plane-remember`` (witness texts cut, facts.json cut to the files its callers name, the repository +URL replaced: the population is reserved, ``tests/test_independent_population.py``).""" + +from __future__ import annotations + +import hashlib +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from openultrasast import cli +from openultrasast.plane import memory +from openultrasast.plane.memory import FileStore, MemoryStoreError, ingest, row_id +from openultrasast.plane.tasks import remember +from openultrasast.plane.tasks.remember import Context, candidates_digest, remember_run, rows_for + +FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46" +CASE = "lmdeploy-media-url-ssrf" +PIN = "71648d239c978452720a6d6692fe914efc8a7b94" +IMAGE = "sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab" + + +@pytest.fixture +def run_dir(tmp_path: Path) -> Path: + target = tmp_path / "plane" / "validation-46" + shutil.copytree(FIXTURE, target) + return target + + +def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]: + rows: dict[str, list[dict[str, Any]]] = {} + for record in store.rows(): + rows.setdefault(record.row["kind"], []).append(record.row) + return rows + + +def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None: + facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes() + assert len(facts) > 100, "the fixture was read" + store = FileStore(tmp_path / "memory") + results = remember_run(run_dir, store, population="population-v2", split="validation") + assert [(r.task, r.skipped, r.kinds) for r in results] == [(f"{CASE}-remember", False, {"facts": 1, "unit_cost": 2, "verdict": 1})] + rows = _by_kind(store) + for row in (r for kind in rows.values() for r in kind): + assert (row["repo"], row["pin"], row["run"], row["image"]) == ("example.com/reserved/lmdeploy", PIN, "validation-46", IMAGE) + assert (row["task"], row["population"], row["split"]) == (f"{CASE}-remember", "population-v2", "validation") + sha = hashlib.sha256(facts).hexdigest() + assert rows["facts"][0]["sha256"] == sha and store.get_facts(sha) == facts + assert rows["facts"][0]["candidates_digest"] == candidates_digest(json.loads(facts)) + agreed = json.loads((run_dir / f"{CASE}-final" / "agreed.json").read_text())["candidates"] + assert [(v["candidate"], v["passes"], v["site_match"], v["usd"], v["turns"]) for v in rows["verdict"]] == [ + (c["candidate"], {"a": c["a"], "b": c["b"], "c": c["c"]}, c["site_match"], c["usd"], c["turns"]) for c in agreed + ], "a verdict row per agree row, from the final (tie-break) decision" + assert rows["verdict"][0]["final"] == "agreed" and rows["verdict"][0]["family"] == "untrusted_destination" + costs = {(u["pass"], u["path"]): (u["usd"], u["calls"], u["model"]) for u in rows["unit_cost"]} + assert costs == { + ("a", "lmdeploy/vl/media/connection.py"): (0.01237569, 7, "deepseek-flash"), + ("b", "lmdeploy/vl/media/connection.py"): (costs[("b", "lmdeploy/vl/media/connection.py")][0], 7, "deepseek-flash"), + } + assert sum(u["usd"] for u in rows["unit_cost"]) == pytest.approx(rows["verdict"][0]["usd"], abs=1e-6) + assert not list(run_dir.glob("*-remember")), "the by-hand path writes nothing into the run directory" + + +def test_re_ingest_of_the_same_run_is_a_no_op(run_dir: Path, tmp_path: Path) -> None: + store = FileStore(tmp_path / "memory") + remember_run(run_dir, store) + before = {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()} + again = remember_run(run_dir, store) + assert [r.skipped for r in again] == [True] + assert {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()} == before + + +def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None: + ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64) + agreed = { + "family": "injection", + "candidates": [ + {"candidate": "a.py::f", "a": True, "b": True, "c": None, "agreed": True}, + {"candidate": "a.py::g", "a": True, "b": False, "c": True, "agreed": True}, + {"candidate": "a.py::h", "a": True, "b": False, "c": False, "agreed": False}, + {"candidate": "a.py::i", "a": False, "b": False, "c": None, "agreed": False}, + ], + "disputed": [{"candidate": "a.py::h"}], + } + alert = {"rule_id": "py-ssrf", "rule_status": "shadow", "path": "a.py", "line": 3, "function": "f", "pin_role": "vulnerable"} + rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]) + verdicts = {r["candidate"]: (r["final"], r["tiebreak"]) for r in rows if r["kind"] == "verdict"} + expected = {"a.py::f": ("agreed", False), "a.py::g": ("agreed", True), "a.py::h": ("disputed", True), "a.py::i": ("rejected", False)} + assert verdicts == expected + (alert_row,) = [r for r in rows if r["kind"] == "alert"] + assert {k: alert_row[k] for k in alert} == alert and alert_row["repo"] == "github.com/o/r" + assert alert_row["id"] == row_id("alert", "r", "c-remember", "py-ssrf:a.py:3") + assert len({r["id"] for r in rows}) == len(rows) + assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic" + + +def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None: + docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text())) + task = next(d for d in docs if d["kind"] == "Task") + pins = next(e["value"] for e in task["spec"]["env"] if e["name"] == "OUSAST_GIT_PINS") + out = run_dir / f"{CASE}-remember" + env = { + "OUSAST_OUTPUT_DIR": str(out), "OUSAST_RUN": "validation-46", "OUSAST_TASK": f"{CASE}-remember", "OUSAST_GIT_PINS": pins, + "AX_TASK_YAML": yaml.safe_dump(task), "AX_WORKSPACES_YAML": yaml.safe_dump_all([d for d in docs if d["kind"] == "Workspace"]), + "OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"), + "OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"), + "OUSAST_POPULATION": "population-v2", "OUSAST_SPLIT": "validation", + **{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"}, + **{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"}, + } # fmt: skip + assert remember.main(env) == 0 + summary = json.loads((out / "summary.json").read_text()) + assert (summary["status"], summary["units_done"], summary["usd"], summary["calls"], summary["rows"]) == ("done", 1, 0, 0, 4) + assert (out / "facts.json").read_bytes() == (run_dir / f"{CASE}-facts" / "facts.json").read_bytes() + delivered = FileStore(tmp_path / "delivered") + assert [r.task for r in ingest(run_dir, delivered)] == [f"{CASE}-remember"] + by_hand = FileStore(tmp_path / "by-hand") + (out / "memory.jsonl").rename(tmp_path / "memory.jsonl") + remember_run(run_dir, by_hand, population="population-v2", split="validation") + assert [r.row for r in delivered.rows()] == [r.row for r in by_hand.rows()], "the task and the host path agree" + assert remember.main({**env, "AX_TASK_YAML": "{}"}) == 2 + assert "names no image" in json.loads((out / "summary.json").read_text())["reason"] + + +def test_ingest_without_remember_outputs_is_a_no_op(run_dir: Path, tmp_path: Path) -> None: + store = FileStore(tmp_path / "memory") + assert ingest(run_dir, store) == [] and not store.root.exists() + + +def test_a_malformed_delivered_row_fails_naming_file_and_line(run_dir: Path, tmp_path: Path) -> None: + out = run_dir / f"{CASE}-remember" + out.mkdir() + good = rows_for(Context("validation-46", out.name, "https://github.com/o/r", PIN, IMAGE), facts=None, passes={}, models={}, agreed=None, + alerts=[{"rule_id": "x", "path": "a.py", "line": 1}]) # fmt: skip + (out / "memory.jsonl").write_text(json.dumps(good[0]) + "\n" + json.dumps({**good[0], "kind": "vibes"}) + "\n") + store = FileStore(tmp_path / "memory") + with pytest.raises(MemoryStoreError, match=rf"{out.name}/memory.jsonl:2: unknown kind"): + remember_run(run_dir, store) + assert not store.root.exists() + + +def test_plane_remember_cli_reads_population_from_the_run_and_reports_counts( + run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path)) + monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory") + assert cli.main(["plane", "remember", "validation-46"]) == 0 + out = capsys.readouterr().out + assert "1 tasks ingested, 0 already in the index; rows facts=1 unit_cost=2 verdict=1" in out + rows = FileStore(tmp_path / "memory").rows() + assert {(r.row["population"], r.row["split"]) for r in rows} == {("population-v2", "validation")} # plane/runs/validation-46.yaml + assert cli.main(["plane", "remember", "validation-46"]) == 0 + assert "0 tasks ingested, 1 already in the index" in capsys.readouterr().out + + +def test_plane_run_seeds_before_and_ingests_after(run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from openultrasast.plane import reconciler + + monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path)) + monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory") + calls: list[str] = [] + monkeypatch.setattr(memory, "seed", lambda manifest, store: calls.append(f"seed {manifest.name}") or []) + monkeypatch.setattr(reconciler, "run", lambda manifest, **kw: calls.append("run") or "done") + assert cli.main(["plane", "run", "plane/runs/validation-46.yaml"]) == 0 + assert calls == ["seed validation-46.yaml", "run"] + assert len(FileStore(tmp_path / "memory").rows(kind="verdict")) == 1, "the run's rows were ingested after it" diff --git a/uv.lock b/uv.lock index b8880b1..ede1954 100644 --- a/uv.lock +++ b/uv.lock @@ -216,6 +216,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, ] +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5f/f652055e18d2627e2eed94c7f31a792127cfe38df786635395d742321674/argon2_cffi_bindings-26.1.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083", size = 15434, upload-time = "2026-08-20T07:32:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, + { url = "https://files.pythonhosted.org/packages/66/cb/214092c39c4dbcb72cf98b12234ddac2221f8fe2c0acf29c6a70fa83be53/argon2_cffi_bindings-26.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb", size = 25683, upload-time = "2026-08-20T07:33:05.337Z" }, + { url = "https://files.pythonhosted.org/packages/83/e5/02015b83e9b05ccb85ff2ced424cf6e83a12d3810bc7f66d679a92b69ffb/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6", size = 27310, upload-time = "2026-08-20T07:33:06.344Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/85e612787d0796878b3b4f6bd53dcd5484b6fe7b64cc6fc7b6e6a04cf835/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990", size = 26771, upload-time = "2026-08-20T07:33:07.429Z" }, + { url = "https://files.pythonhosted.org/packages/f6/84/ccb003b6f9969820e87656398f4d49c857def71a85ca1588a0e809afd7ce/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08", size = 27569, upload-time = "2026-08-20T07:33:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/c26b76debf0998ee08fbe947ab2058ac5de37d4b9d46b06c17abaa6c4ce9/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca", size = 27279, upload-time = "2026-08-20T07:33:09.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0d/ead6ddc029f91bc9b9390686dad3c808ab08100d348f6266b5f93f8970ee/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1", size = 27774, upload-time = "2026-08-20T07:33:10.728Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/c108530d9eb86036b78d3af4de28b83b4a2d9a70512bd10ff8e59966aab4/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36", size = 26933, upload-time = "2026-08-20T07:33:11.661Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/0bfc59e781c89acf64c31c388aade9d9d1c1ea38aa1ba1292fe07f607fe9/argon2_cffi_bindings-26.1.0-cp315-cp315t-win32.whl", hash = "sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210", size = 24875, upload-time = "2026-08-20T07:33:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/61/c7/c3e46068cddffccecb8ad94d71135e9bf62bbc789589e7dfadc7c6f59214/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4", size = 26655, upload-time = "2026-08-20T07:33:13.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ca/18b9c8c45fecf34b9100ec6d7946057f14a158f2eaa20ea123a3e82351cb/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440", size = 25376, upload-time = "2026-08-20T07:33:14.491Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b9/97f0370f99611b14efd384918613dd5cbda75f28d9bb1b677aacfeaa17df/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8", size = 23055, upload-time = "2026-08-20T07:33:19.716Z" }, + { url = "https://files.pythonhosted.org/packages/ae/70/7eb3fe7bf00103cbbb569c51aef150661f22b734a782673a600ff0f52309/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a", size = 24869, upload-time = "2026-08-20T07:33:20.671Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4b/9d5919c6cb1f15df7406af0f99b048bd93936f112e3e8f4c8077bc2a9110/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba", size = 24216, upload-time = "2026-08-20T07:33:21.653Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" }, +] + [[package]] name = "ast-serialize" version = "0.5.0" @@ -1765,6 +1824,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "minio" +version = "7.2.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -2140,11 +2215,17 @@ wheels = [ name = "openultrasast" version = "1.2.0a1" source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] [package.optional-dependencies] harnessx = [ { name = "harnessx" }, ] +minio = [ + { name = "minio" }, +] semantic = [ { name = "tree-sitter" }, { name = "tree-sitter-c" }, @@ -2167,6 +2248,8 @@ dev = [ [package.metadata] requires-dist = [ { name = "harnessx", marker = "extra == 'harnessx'", git = "https://github.com/Darwin-Agent/HarnessX.git?rev=274c512eea67c64701f992d3b6fe06f0fd590ef0" }, + { name = "minio", marker = "extra == 'minio'", specifier = ">=7.2,<8" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "tree-sitter", marker = "extra == 'semantic'", specifier = ">=0.23,<0.27" }, { name = "tree-sitter-c", marker = "extra == 'semantic'", specifier = ">=0.23" }, { name = "tree-sitter-cpp", marker = "extra == 'semantic'", specifier = ">=0.23" }, @@ -2175,7 +2258,7 @@ requires-dist = [ { name = "tree-sitter-python", marker = "extra == 'semantic'", specifier = ">=0.23" }, { name = "tree-sitter-typescript", marker = "extra == 'semantic'", specifier = ">=0.23" }, ] -provides-extras = ["harnessx", "semantic"] +provides-extras = ["harnessx", "minio", "semantic"] [package.metadata.requires-dev] dev = [ @@ -2518,6 +2601,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + [[package]] name = "pydantic" version = "2.13.4"