mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): memory store (file and MinIO) and the remember task (harnessx-removal task 2)
plane/memory.py: the MemoryStore interface with FileStore (index.jsonl, facts by content hash, repos/<host>__<owner>__<name>/<pin>.jsonl, idempotent ingest, deterministic row ids, refusal below 1 GiB free naming the path) and MinioStore (object metadata and tags, S3 Select pushdown probed once per process with a logged local fallback, object version ids on every row read, versioning and runs/ lifecycle helper, presigned URLs). OUSAST_MEMORY selects the backend; MINIO_* settings come from .env; the minio SDK is the optional extra `minio`. plane/tasks/remember.py: model-free rows (facts, verdict, unit_cost, alert) from a case's artifacts; remember_run computes them on the host for a recorded run without writing into it. reconciler.mark_done (477 lines total), seed before and ingest after `ousast plane run`, `ousast plane remember <run>`. The generator annotates each repo-facts Task with openultrasast.io/memory-key (repo, pin, candidates digest, image digest) and the Run with its population and split; plane/ regenerated. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
d64ab2817e
commit
52d28b5121
@@ -12,7 +12,7 @@ and passed its fixture test. Every task is a commit gated on the full suite's ow
|
||||
written next to it.
|
||||
- _Requirements: 3.1, 5.1_
|
||||
|
||||
- [ ] 2. Memory store and the `remember` task
|
||||
- [x] 2. Memory store and the `remember` task
|
||||
- `plane/memory.py`: the `MemoryStore` interface with `FileStore` and `MinioStore` (object metadata/tags,
|
||||
S3 Select JSON queries probed with a local fallback, versioned provenance, lifecycle, presign stubs;
|
||||
`OUSAST_MEMORY` selects; MinIO settings from `.env`; `minio` SDK as an optional extra); JSON rows per
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"modules": 132,
|
||||
"modules": 134,
|
||||
"counts": {
|
||||
"load_bearing": 123,
|
||||
"load_bearing": 125,
|
||||
"standalone_capability": 9
|
||||
},
|
||||
"orphaned": [],
|
||||
@@ -117,6 +117,7 @@
|
||||
"hunter_tools",
|
||||
"model.endpoint",
|
||||
"pairs",
|
||||
"plane.memory",
|
||||
"push.assistance",
|
||||
"push.policy",
|
||||
"push.runner",
|
||||
@@ -663,9 +664,11 @@
|
||||
"cli",
|
||||
"plane.egress",
|
||||
"plane.generate",
|
||||
"plane.memory",
|
||||
"plane.reconciler",
|
||||
"plane.router",
|
||||
"plane.runner",
|
||||
"plane.tasks.remember",
|
||||
"plane.tasks.verify",
|
||||
"plane.workspaces"
|
||||
]
|
||||
@@ -716,12 +719,22 @@
|
||||
"plane.workspaces"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "plane.memory",
|
||||
"classification": "load_bearing",
|
||||
"reason": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`",
|
||||
"importers": [
|
||||
"plane.generate",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "plane.reconciler",
|
||||
"classification": "load_bearing",
|
||||
"reason": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`",
|
||||
"importers": [
|
||||
"plane.generate",
|
||||
"plane.memory",
|
||||
"plane.workspaces"
|
||||
]
|
||||
},
|
||||
@@ -745,8 +758,10 @@
|
||||
"classification": "load_bearing",
|
||||
"reason": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)",
|
||||
"importers": [
|
||||
"cli",
|
||||
"plane.generate",
|
||||
"plane.tasks.agree"
|
||||
"plane.tasks.agree",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -755,11 +770,22 @@
|
||||
"reason": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match",
|
||||
"importers": []
|
||||
},
|
||||
{
|
||||
"module": "plane.tasks.remember",
|
||||
"classification": "load_bearing",
|
||||
"reason": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free",
|
||||
"importers": [
|
||||
"cli"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "plane.tasks.repo_facts",
|
||||
"classification": "load_bearing",
|
||||
"reason": "ai-service-plane Req 5.1/5.2: source-only functions and callers per file; runner integration pending",
|
||||
"importers": []
|
||||
"importers": [
|
||||
"plane.generate",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "plane.tasks.verify",
|
||||
|
||||
@@ -4,6 +4,9 @@ apiVersion: openultrasast.io/v1alpha1
|
||||
kind: Run
|
||||
metadata:
|
||||
name: validation-46
|
||||
annotations:
|
||||
openultrasast.io/population: population-v2
|
||||
openultrasast.io/split: validation
|
||||
spec:
|
||||
tasks:
|
||||
- name: budibase-mongo-template-nosqli-facts
|
||||
|
||||
@@ -4,6 +4,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-budibase-mongo-template-nosqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"b4ef709df34ab08c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b7e63df737eede85d951aa9665b0c0778ece7ccb","repo":"github.com/Budibase/budibase"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -120,6 +122,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-cct-studio-server-cmdi
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"6c9fa4a6e47cda37","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"acdb5893412a81eaec9862ee24a4613bf296ab45","repo":"github.com/davila7/claude-code-templates"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -236,6 +240,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-contextforge-prompt-ssti
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"b272af6456664ebc","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"0f8e0ad679f1fff685f486e17f642ac863c8ca9e","repo":"github.com/IBM/mcp-context-forge"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -352,6 +358,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-froxlor-admin-ip-second-order-sqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"c52240d980ef0105","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9bf03de643a2edee4642bfc6c94db1f05253da02","repo":"github.com/froxlor/froxlor"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -468,6 +476,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-fuxa-tdengine-escape-sqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"ec699ee54d38a8ee","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"193a888f8e3f805325072829b1644a2c768c948b","repo":"github.com/frangoteam/FUXA"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -584,6 +594,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-lmdeploy-media-url-ssrf
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"1337c9c717f5ac87","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"71648d239c978452720a6d6692fe914efc8a7b94","repo":"github.com/InternLM/lmdeploy"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -700,6 +712,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-medplum-external-auth-open-redirect
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"17ba609d5b1a1a00","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"5d4a8654b02aa67c444f96ae1eb5663c09854006","repo":"github.com/medplum/medplum"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -816,6 +830,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-pgadmin-maintenance-sqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"7f1b83a88024c2d4","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"d112dc3b962218efc3613fa727d54f0fc5b1d527","repo":"github.com/pgadmin-org/pgadmin4"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -932,6 +948,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-qwed-sympy-eval
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"f071e8db5f5517ab","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"06801f68cbf9c9b5e54cc17f2a61cb2792da7a26","repo":"github.com/QWED-AI/qwed-verification"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1048,6 +1066,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-reactpress-findall-column-sqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"ecb82a10249c6fed","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"c3dd10681b3d2b5e224f7ac647b3769557603975","repo":"github.com/fecommunity/reactpress"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1164,6 +1184,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-unleash-addon-webhook-ssrf
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"4d3debd51fd86f9c","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"b9f3217d327f24094b4352130baa30387f85cffe","repo":"github.com/Unleash/unleash"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1280,6 +1302,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-winml-cli-serve-cors
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"e3b0c44298fc1c14","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"02179d185f0e3ac23518236b5ca1f61fd37ef680","repo":"github.com/microsoft/winml-cli"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1396,6 +1420,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-wp-directorist-avatar-ssrf
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"d6963021f588f39e","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"77cfc9bc8b453f97ab7e720330706fa4ef056867","repo":"github.com/sovware/directorist"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1512,6 +1538,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-wp-gopay-log-filter-sqli
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"626bc8b5f8d33de6eefd921e3a05ed52ac320bcd","repo":"github.com/gopaycommunity/gopay-woocommerce-integration"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
@@ -1628,6 +1656,8 @@ kind: Task
|
||||
metadata:
|
||||
name: repo-facts-wp-groundhogg-confirm-redirect
|
||||
atespace: default
|
||||
annotations:
|
||||
openultrasast.io/memory-key: '{"candidates":"fde4d17144a13d73","image":"sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617","pin":"9472a1dd79d4556c40eece501a61930793f170d3","repo":"github.com/groundhoggwp/groundhogg"}'
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:fb9a743027518976659fd11505e58cc3fc84fb42e07627196e13b76ac01d0617
|
||||
command:
|
||||
|
||||
@@ -20,6 +20,11 @@ dependencies = ["pyyaml>=6.0"]
|
||||
harnessx = [
|
||||
"harnessx @ git+https://github.com/Darwin-Agent/HarnessX.git@274c512eea67c64701f992d3b6fe06f0fd590ef0",
|
||||
]
|
||||
# The plane's memory store on MinIO (OUSAST_MEMORY=minio://<bucket>, plane/memory.py). Imported lazily; the
|
||||
# core install and the file store need nothing beyond PyYAML.
|
||||
minio = [
|
||||
"minio>=7.2,<8",
|
||||
]
|
||||
# Structured parse for overlay (JS/C/C++/Java/Python). Core install stays empty.
|
||||
# Wheels are MIT; import is lazy via openultrasast.semantic.extra.
|
||||
semantic = [
|
||||
@@ -74,6 +79,11 @@ ignore_missing_imports = true
|
||||
module = "tree_sitter"
|
||||
ignore_missing_imports = true
|
||||
|
||||
# The minio SDK is an optional extra, absent in the type-check environment (plane/memory.py imports it lazily).
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["minio", "minio.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
# PyYAML ships no inline types and types-PyYAML is not a dev dependency.
|
||||
[[tool.mypy.overrides]]
|
||||
module = "yaml"
|
||||
|
||||
@@ -248,6 +248,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
plane_status.add_argument("run")
|
||||
plane_status.add_argument("--units", action="store_true", help="per-unit rows from each task's units.jsonl")
|
||||
plane_sub.add_parser("doctor", help="check kind, Agent Substrate, the ax controller and the runner image")
|
||||
plane_remember = plane_sub.add_parser("remember", help="ingest a run's memory rows into the store (OUSAST_MEMORY)")
|
||||
plane_remember.add_argument("run")
|
||||
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
|
||||
plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)")
|
||||
plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)")
|
||||
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
|
||||
plane_ws.add_argument("population", type=Path)
|
||||
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
|
||||
@@ -1304,9 +1309,14 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
from .plane import reconciler # lazy: the plane is a maintainer surface, not the scan path
|
||||
|
||||
if args.plane_command == "run":
|
||||
_plane_memory("seed", args.run_manifest)
|
||||
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax)
|
||||
print(f"run finished: {result}")
|
||||
_plane_memory("ingest", args.run_manifest)
|
||||
return 0 if result == "done" else 1
|
||||
if args.plane_command == "remember":
|
||||
manifest = args.plane / "runs" / f"{args.run}.yaml"
|
||||
return _plane_memory("ingest", manifest if manifest.is_file() else None, run=args.run, population=args.population, split=args.split)
|
||||
if args.plane_command == "status":
|
||||
print(reconciler.status(args.run, units=args.units))
|
||||
return 0
|
||||
@@ -1324,6 +1334,40 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_memory(
|
||||
step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None
|
||||
) -> int:
|
||||
"""``seed`` before a Run, ``ingest`` after it (and ``ousast plane remember``). A store failure is reported and
|
||||
never changes the run's result: ingest can be repeated by hand."""
|
||||
from .plane import memory, reconciler
|
||||
from .plane.tasks.remember import UNKNOWN, remember_run
|
||||
|
||||
try:
|
||||
store = memory.open_store()
|
||||
if step == "seed":
|
||||
assert run_manifest is not None
|
||||
seeded = memory.seed(run_manifest, store)
|
||||
if seeded:
|
||||
print(f"memory {store.describe()}: facts reused for {len(seeded)} tasks: {', '.join(seeded)}")
|
||||
return 0
|
||||
annotations: dict[str, str] = {}
|
||||
if run_manifest is not None:
|
||||
spec, _ = reconciler.load_run(run_manifest)
|
||||
run, annotations = run or spec.metadata.name, dict(spec.metadata.annotations)
|
||||
assert run is not None
|
||||
results = remember_run(
|
||||
reconciler.run_dir(run), store, population=population or annotations.get(memory.POPULATION_ANNOTATION) or UNKNOWN,
|
||||
split=split or annotations.get(memory.SPLIT_ANNOTATION) or UNKNOWN,
|
||||
) # fmt: skip
|
||||
except (memory.MemoryStoreError, OSError, ValueError) as exc:
|
||||
print(f"memory {step} failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
fresh = [r for r in results if not r.skipped]
|
||||
kinds = " ".join(f"{k}={n}" for k, n in memory.counts(fresh).items()) or "none"
|
||||
print(f"memory {store.describe()}: {run}: {len(fresh)} tasks ingested, {len(results) - len(fresh)} already in the index; rows {kinds}")
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_increment(args: argparse.Namespace) -> int:
|
||||
from .plane.generate import increment
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ SPEC_OWNED = {
|
||||
"plane.tasks.agree": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match",
|
||||
"plane.reconciler": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`",
|
||||
"plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set",
|
||||
"plane.memory": "harnessx-removal Req 6.1/6.4: the memory store (file, MinIO), ingest and fact reuse; `ousast plane remember`",
|
||||
"plane.tasks.remember": "harnessx-removal Req 6.1: a case's delivered artifacts as memory rows, model-free",
|
||||
"plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`",
|
||||
"plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory",
|
||||
}
|
||||
|
||||
@@ -15,7 +15,11 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
|
||||
(``triage.json``, for the measurement);
|
||||
- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
|
||||
Workspaces and naming the files as env;
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets.
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets, annotated with the
|
||||
population and split (``openultrasast.io/population``/``split``: the files' stems) the memory rows carry.
|
||||
|
||||
Each ``repo-facts`` Task carries ``openultrasast.io/memory-key`` (repository, pin, candidates digest, runner image
|
||||
digest): the key under which stored facts are reused instead of recomputed (``plane/memory.py``, ``seed``).
|
||||
|
||||
Why per-case Tasks rather than one Task per step: an ax Task binds its Workspaces by name and a Run entry adds
|
||||
neither a binding nor env (it names a Task, artifacts and a budget), and ``Run.inputs`` accepts only
|
||||
@@ -53,7 +57,9 @@ import yaml
|
||||
from ..model.endpoint import price_of
|
||||
from .budget import cost_of
|
||||
from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
|
||||
from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, memory_key
|
||||
from .reconciler import _ax_name, _doc
|
||||
from .tasks.repo_facts import _digest
|
||||
from .tasks.verify import MAX_STEPS, units_of
|
||||
|
||||
__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
|
||||
@@ -251,8 +257,11 @@ def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]:
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest."""
|
||||
def render(
|
||||
cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str, *, population: str = "", split: str = ""
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
|
||||
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry."""
|
||||
header = f"# generated by `{command}`; do not edit, regenerate.\n"
|
||||
out: dict[str, str] = {}
|
||||
tasks: list[dict[str, Any]] = []
|
||||
@@ -266,7 +275,11 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name:
|
||||
repo = [(pinned["metadata"]["name"], CASE_PATH), (inputs["metadata"]["name"], INPUTS_PATH)]
|
||||
on_repo = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"}
|
||||
facts_env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/functions.json"}
|
||||
tasks.append(_task(templates["repo-facts"], f"repo-facts-{case_id}", facts_env, repo))
|
||||
facts_task = _task(templates["repo-facts"], f"repo-facts-{case_id}", facts_env, repo)
|
||||
digest = _digest(sorted({fn for _, fn, _ in item.kept}))
|
||||
key = memory_key(str(item.case["repo"]), str(item.case["vulnerable"]), digest, templates["repo-facts"].image or "")
|
||||
facts_task["metadata"].setdefault("annotations", {})[MEMORY_KEY_ANNOTATION] = key
|
||||
tasks.append(facts_task)
|
||||
for label in ("a", "b"):
|
||||
env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json", "OUSAST_PASS": label}
|
||||
tasks.append(_task(templates["verify"], f"verify-{label}-{case_id}", env, repo))
|
||||
@@ -295,7 +308,9 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name:
|
||||
tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first
|
||||
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
|
||||
out[f"tasks/{run_name}.yaml"] = _dump(header, tasks)
|
||||
run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": {"name": run_name}, "spec": {"tasks": entries}}
|
||||
annotations = {k: v for k, v in ((POPULATION_ANNOTATION, population), (SPLIT_ANNOTATION, split)) if v}
|
||||
metadata: dict[str, Any] = {"name": run_name, **({"annotations": annotations} if annotations else {})}
|
||||
run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": metadata, "spec": {"tasks": entries}}
|
||||
total = sum(float(e["budget"]["usd"]) for e in entries)
|
||||
candidates = sum(len(c.candidates) for c in cases)
|
||||
kept = sum(len(c.kept) for c in cases)
|
||||
@@ -340,7 +355,8 @@ def increment(
|
||||
if runner_image is not None:
|
||||
repin_templates(plane, runner_image)
|
||||
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
|
||||
files = render(read_cases(validation_set, candidates_dir, triage_dir, population, repos), loaded.tasks, run_name, command)
|
||||
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos)
|
||||
files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem)
|
||||
written: list[Path] = []
|
||||
for relative, text in files.items():
|
||||
path = plane / relative
|
||||
|
||||
@@ -0,0 +1,695 @@
|
||||
"""The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4).
|
||||
|
||||
One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows
|
||||
of five kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``proposal_outcome``) that the ``remember`` task
|
||||
(:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends::
|
||||
|
||||
index.jsonl one row per ingested (run, task): sha256 of its rows, row count
|
||||
facts/<sha256>.json facts.json by content (identical facts stored once)
|
||||
repos/<host>__<owner>__<name>/<pin>.jsonl the rows of one repository + 40-hex pin, sorted by id
|
||||
|
||||
``OUSAST_MEMORY`` selects the backend: ``file:///path`` (:class:`FileStore`, default ``results_root()/memory``) or
|
||||
``minio://<bucket>[/<prefix>]`` (:class:`MinioStore`, the ``minio`` extra). MinIO's endpoint and credentials come
|
||||
from ``.env`` (``MINIO_ENDPOINT``, ``MINIO_ACCESS_KEY``, ``MINIO_SECRET_KEY``, ``MINIO_SECURE``), never from a
|
||||
manifest, and are never printed. Ingest is idempotent: an ``index.jsonl`` hit on (run, task, sha256) skips the
|
||||
delivery, and rows carry a deterministic ``id``, so a repeated row replaces itself. Every object is written whole
|
||||
(``FileStore``: a temporary file renamed) and the index last, so an interrupted ingest leaves nothing partial that
|
||||
a repeat would not repair. ``FileStore`` refuses to write below 1 GiB free and names its path.
|
||||
|
||||
Fact reuse: a facts entry is valid for (repo, pin, candidates digest, runner image digest), the key the generator
|
||||
writes as the Task annotation ``openultrasast.io/memory-key``. :func:`seed` runs before a Run: for every entry whose
|
||||
Task carries a key with stored facts and no state yet, it writes ``<run>/<task>/facts.json`` and a ``done``
|
||||
summary and marks the task done under the run lock (``reconciler.mark_done``), so the reconciler skips it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
|
||||
POPULATION_ANNOTATION = "openultrasast.io/population"
|
||||
SPLIT_ANNOTATION = "openultrasast.io/split"
|
||||
KINDS = ("facts", "verdict", "unit_cost", "alert", "proposal_outcome")
|
||||
ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
|
||||
TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
|
||||
MIN_FREE_BYTES = 1 << 30
|
||||
MINIO_KEYS = ("MINIO_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY")
|
||||
_PIN = re.compile(r"[0-9a-f]{40}")
|
||||
_FIELD = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
_TAG_VALUE = re.compile(r"[^A-Za-z0-9 +\-=._:/@]")
|
||||
|
||||
|
||||
class MemoryStoreError(RuntimeError):
|
||||
"""The store refused or failed a read or write; the message names the store and the cause."""
|
||||
|
||||
|
||||
# --- rows and keys -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def repo_key(repo: str) -> str:
|
||||
"""``host/owner/name`` of a repository URL (scheme, credentials, ``.git`` and trailing slashes dropped)."""
|
||||
text = repo.strip()
|
||||
if "://" in text:
|
||||
parts = urlsplit(text)
|
||||
text = f"{parts.hostname or ''}{parts.path}"
|
||||
text = text.rstrip("/").removesuffix(".git").strip("/")
|
||||
if not text:
|
||||
raise ValueError(f"not a repository: {repo!r}")
|
||||
return text
|
||||
|
||||
|
||||
def repo_dir(repo: str) -> str:
|
||||
return repo_key(repo).replace("/", "__")
|
||||
|
||||
|
||||
def row_id(kind: str, run: str, task: str, subject: str) -> str:
|
||||
"""The deterministic row id: sha256 of (kind, run, task, subject), subject being the candidate, the path,
|
||||
the facts sha256 or the rule id with its location."""
|
||||
return hashlib.sha256("\x00".join((kind, run, task, subject)).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def image_digest(image: str) -> str:
|
||||
"""The content part of an image reference (``sha256:...``): the registry host may differ, the code may not."""
|
||||
return image.rpartition("@")[2] if "@" in image else image
|
||||
|
||||
|
||||
def memory_key(repo: str, pin: str, candidates: str, image: str) -> str:
|
||||
"""The annotation text of a facts entry's validity key."""
|
||||
key = {"candidates": candidates, "image": image_digest(image), "pin": pin, "repo": repo_key(repo)}
|
||||
return json.dumps(key, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def parse_memory_key(text: str) -> dict[str, str]:
|
||||
key = json.loads(text)
|
||||
if not isinstance(key, dict) or set(key) != {"candidates", "image", "pin", "repo"}:
|
||||
raise ValueError(f"{MEMORY_KEY_ANNOTATION} must name candidates, image, pin and repo: {text[:200]!r}")
|
||||
return {k: str(v) for k, v in key.items()}
|
||||
|
||||
|
||||
def validate_row(row: object, where: str = "row") -> dict[str, Any]:
|
||||
if not isinstance(row, dict):
|
||||
raise MemoryStoreError(f"{where}: a row must be a JSON object")
|
||||
missing = [f for f in ROW_FIELDS if not isinstance(row.get(f), str) or not row.get(f)]
|
||||
if missing:
|
||||
raise MemoryStoreError(f"{where}: missing or empty fields {missing}")
|
||||
if row["kind"] not in KINDS:
|
||||
raise MemoryStoreError(f"{where}: unknown kind {row['kind']!r} (one of {', '.join(KINDS)})")
|
||||
if not _PIN.fullmatch(row["pin"]):
|
||||
raise MemoryStoreError(f"{where}: pin {row['pin']!r} is not a 40-hex commit")
|
||||
return row
|
||||
|
||||
|
||||
def _line(row: Mapping[str, Any]) -> str:
|
||||
return json.dumps(row, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def rows_digest(rows: Iterable[Mapping[str, Any]]) -> str:
|
||||
return hashlib.sha256("".join(_line(r) + "\n" for r in rows).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def matches(row: Mapping[str, Any], where: Mapping[str, Any] | None) -> bool:
|
||||
"""Top-level equality on every ``where`` field; the rule the S3 Select expression pushes down."""
|
||||
return all(row.get(k) == v for k, v in (where or {}).items())
|
||||
|
||||
|
||||
def where_sql(where: Mapping[str, Any]) -> str:
|
||||
"""The S3 Select expression of ``where`` (JSON Lines input): equality per field, AND-joined."""
|
||||
clauses: list[str] = []
|
||||
for name, value in sorted(where.items()):
|
||||
if not _FIELD.fullmatch(name):
|
||||
raise ValueError(f"where: field {name!r} is not a plain identifier")
|
||||
ref = f's."{name}"'
|
||||
if value is None:
|
||||
clauses.append(f"{ref} IS NULL")
|
||||
elif isinstance(value, bool):
|
||||
clauses.append(f"{ref} = {'true' if value else 'false'}")
|
||||
elif isinstance(value, int | float):
|
||||
clauses.append(f"{ref} = {value!r}")
|
||||
elif isinstance(value, str):
|
||||
clauses.append(f"{ref} = '{value.replace(chr(39), chr(39) * 2)}'")
|
||||
else:
|
||||
raise ValueError(f"where: {name} must be a string, number, boolean or null")
|
||||
return "SELECT * FROM S3Object s" + (" WHERE " + " AND ".join(clauses) if clauses else "")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Record:
|
||||
"""A row as read, with the object it came from and that object's version (Req 6.4 provenance)."""
|
||||
|
||||
row: dict[str, Any]
|
||||
key: str
|
||||
version: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IngestResult:
|
||||
run: str
|
||||
task: str
|
||||
rows: int
|
||||
skipped: bool
|
||||
kinds: dict[str, int]
|
||||
|
||||
|
||||
# --- the interface -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MemoryStore(ABC):
|
||||
"""Rows by repository and pin, facts by content hash, and the ingest index; two backends, one behaviour."""
|
||||
|
||||
# primitives a backend implements; keys are the layout's paths
|
||||
@abstractmethod
|
||||
def _get(self, key: str) -> tuple[bytes, str | None] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def _delete(self, key: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def _keys(self, prefix: str) -> list[str]: ...
|
||||
|
||||
@abstractmethod
|
||||
def describe(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: ...
|
||||
|
||||
def _check_space(self) -> None: # noqa: B027 -- optional: only a backend with a local disk overrides it
|
||||
"""A backend with a local disk refuses a write that would fill it."""
|
||||
|
||||
def _select(self, key: str, where: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None] | None:
|
||||
"""Rows of ``key`` matching ``where``, filtered where the data lives; None when the object is absent."""
|
||||
found = self._get(key)
|
||||
if found is None:
|
||||
return None
|
||||
return [r for r in _parse_lines(found[0], key) if matches(r, where)], found[1]
|
||||
|
||||
def _object_may_match(self, key: str, kind: str | None) -> bool:
|
||||
return True
|
||||
|
||||
# --- reads ---
|
||||
def index(self) -> list[dict[str, Any]]:
|
||||
found = self._get("index.jsonl")
|
||||
return _parse_lines(found[0], "index.jsonl") if found else []
|
||||
|
||||
def get_facts(self, sha256: str) -> bytes | None:
|
||||
"""The facts.json stored under ``sha256``; a file whose content no longer hashes to it is dropped with a
|
||||
warning and reported absent, so the task recomputes."""
|
||||
key = f"facts/{sha256}.json"
|
||||
found = self._get(key)
|
||||
if found is None:
|
||||
return None
|
||||
if hashlib.sha256(found[0]).hexdigest() != sha256:
|
||||
log.warning("memory %s: %s no longer matches its sha256; dropped, the task recomputes", self.describe(), key)
|
||||
self._delete(key)
|
||||
return None
|
||||
return found[0]
|
||||
|
||||
def rows(
|
||||
self, repo: str | None = None, pin: str | None = None, kind: str | None = None, where: Mapping[str, Any] | None = None
|
||||
) -> list[Record]:
|
||||
"""Rows filtered by repository, pin, kind and top-level equality on ``where``, in key then id order."""
|
||||
prefix = f"repos/{repo_dir(repo)}/" if repo else "repos/"
|
||||
keys = [k for k in self._keys(prefix) if k.endswith(".jsonl") and (pin is None or k.endswith(f"/{pin}.jsonl"))]
|
||||
wanted = {**(where or {}), **({"kind": kind} if kind else {})}
|
||||
records: list[Record] = []
|
||||
for key in sorted(keys):
|
||||
if not self._object_may_match(key, kind):
|
||||
continue
|
||||
selected = self._select(key, wanted)
|
||||
if selected is not None:
|
||||
records.extend(Record(r, key, selected[1]) for r in sorted(selected[0], key=lambda r: str(r["id"])))
|
||||
return records
|
||||
|
||||
# --- writes ---
|
||||
def put_facts(self, data: bytes) -> str:
|
||||
sha = hashlib.sha256(data).hexdigest()
|
||||
key = f"facts/{sha}.json"
|
||||
if self._get(key) is None:
|
||||
self._check_space()
|
||||
self._put(key, data)
|
||||
return sha
|
||||
|
||||
def put_row(self, row: Mapping[str, Any]) -> None:
|
||||
self.put_rows([row])
|
||||
|
||||
def put_rows(self, rows: Sequence[Mapping[str, Any]]) -> None:
|
||||
"""Merge rows into their repository + pin objects by ``id`` (a repeated id replaces the stored row)."""
|
||||
groups: dict[str, dict[str, dict[str, Any]]] = {}
|
||||
for index, row in enumerate(rows):
|
||||
checked = validate_row(dict(row), f"row {index}")
|
||||
groups.setdefault(f"repos/{repo_dir(checked['repo'])}/{checked['pin']}.jsonl", {})[checked["id"]] = checked
|
||||
if groups:
|
||||
self._check_space()
|
||||
for key, new in sorted(groups.items()):
|
||||
found = self._get(key)
|
||||
merged = {str(r["id"]): r for r in (_parse_lines(found[0], key) if found else [])}
|
||||
merged.update(new)
|
||||
ordered = [merged[i] for i in sorted(merged)]
|
||||
self._put(key, "".join(_line(r) + "\n" for r in ordered).encode("utf-8"), _labels(ordered))
|
||||
|
||||
def ingest_rows(self, run: str, task: str, rows: Sequence[Mapping[str, Any]], facts: bytes | None = None) -> IngestResult:
|
||||
"""One (run, task)'s rows and facts; a repeat of the same rows is skipped by the index."""
|
||||
checked = [validate_row(dict(r), f"{run}/{task} row {i + 1}") for i, r in enumerate(rows)]
|
||||
digest = rows_digest(checked)
|
||||
kinds: dict[str, int] = {}
|
||||
for row in checked:
|
||||
kinds[row["kind"]] = kinds.get(row["kind"], 0) + 1
|
||||
index = self.index()
|
||||
if any(e.get("run") == run and e.get("task") == task and e.get("sha256") == digest for e in index):
|
||||
return IngestResult(run, task, len(checked), True, kinds)
|
||||
self._check_space()
|
||||
if facts is not None:
|
||||
self.put_facts(facts)
|
||||
self.put_rows(checked)
|
||||
entry = {"run": run, "task": task, "sha256": digest, "rows": len(checked), "ingested": _now()}
|
||||
self._put("index.jsonl", "".join(_line(e) + "\n" for e in [*index, entry]).encode("utf-8"))
|
||||
return IngestResult(run, task, len(checked), False, kinds)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _parse_lines(data: bytes, where: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for number, raw in enumerate(data.decode("utf-8").splitlines(), 1):
|
||||
if not raw.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(raw)
|
||||
except ValueError as exc:
|
||||
raise MemoryStoreError(f"{where}:{number}: not JSON ({exc})") from exc
|
||||
if not isinstance(row, dict):
|
||||
raise MemoryStoreError(f"{where}:{number}: not a JSON object")
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _labels(rows: Sequence[Mapping[str, Any]]) -> dict[str, str]:
|
||||
"""Object metadata/tags: per field the distinct values of the object's rows, space-joined, S3-safe, <=256."""
|
||||
labels: dict[str, str] = {}
|
||||
for name in TAG_FIELDS:
|
||||
values = sorted({_TAG_VALUE.sub("_", str(r[name])) for r in rows if r.get(name) not in (None, "")})
|
||||
text = " ".join(values)
|
||||
labels[name] = text if len(text) <= 256 else "*" # "*": too many to list; the object may match anything
|
||||
return labels
|
||||
|
||||
|
||||
# --- the file backend ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FileStore(MemoryStore):
|
||||
"""The layout under a directory: this laptop's store and the unit tests'."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"file://{self.root}"
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
path = (self.root / key).resolve()
|
||||
if not path.is_relative_to(self.root.resolve()):
|
||||
raise MemoryStoreError(f"{key!r} escapes the store {self.root}")
|
||||
return path
|
||||
|
||||
def _check_space(self) -> None:
|
||||
probe = self.root
|
||||
while not probe.exists() and probe != probe.parent:
|
||||
probe = probe.parent
|
||||
free = shutil.disk_usage(probe).free
|
||||
if free < MIN_FREE_BYTES:
|
||||
raise MemoryStoreError(
|
||||
f"refusing to write the memory store at {self.root}: {free / (1 << 30):.2f} GiB free, below 1 GiB "
|
||||
"(free space, or set OUSAST_MEMORY to another store; `ousast plane remember <run>` repeats the ingest)"
|
||||
)
|
||||
|
||||
def _get(self, key: str) -> tuple[bytes, str | None] | None:
|
||||
path = self._path(key)
|
||||
if not path.is_file():
|
||||
return None
|
||||
data = path.read_bytes()
|
||||
return data, "sha256:" + hashlib.sha256(data).hexdigest() # a file has no version: its content names it
|
||||
|
||||
def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None:
|
||||
path = self._path(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as sink:
|
||||
sink.write(data)
|
||||
os.replace(tmp, path)
|
||||
except BaseException:
|
||||
Path(tmp).unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
def _delete(self, key: str) -> None:
|
||||
self._path(key).unlink(missing_ok=True)
|
||||
|
||||
def _keys(self, prefix: str) -> list[str]:
|
||||
base = self._path(prefix)
|
||||
if not base.is_dir():
|
||||
return []
|
||||
return sorted(p.relative_to(self.root.resolve()).as_posix() for p in base.rglob("*") if p.is_file() and not p.name.startswith("."))
|
||||
|
||||
def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str:
|
||||
return self._path(key).as_uri()
|
||||
|
||||
def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str:
|
||||
return self._path(key).as_uri()
|
||||
|
||||
|
||||
# --- the MinIO backend ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ObjectClient(Protocol):
|
||||
"""The object operations :class:`MinioStore` needs; :class:`SdkClient` adapts the ``minio`` SDK, tests fake it."""
|
||||
|
||||
def get(self, key: str) -> tuple[bytes, str | None] | None: ...
|
||||
def put(self, key: str, data: bytes, labels: Mapping[str, str]) -> None: ...
|
||||
def delete(self, key: str) -> None: ...
|
||||
def keys(self, prefix: str) -> list[str]: ...
|
||||
def tags(self, key: str) -> dict[str, str]: ...
|
||||
def version(self, key: str) -> str | None: ...
|
||||
def select(self, key: str, expression: str) -> bytes: ...
|
||||
def presign(self, method: str, key: str, expires: timedelta) -> str: ...
|
||||
def expire(self, prefix: str, days: int, rule_id: str) -> None: ...
|
||||
def enable_versioning(self) -> None: ...
|
||||
|
||||
|
||||
_SELECT_PROBES: dict[str, bool] = {}
|
||||
PROBE_KEY = "_probe/select.jsonl"
|
||||
|
||||
|
||||
class MinioStore(MemoryStore):
|
||||
"""The layout in one bucket (optionally under a prefix): object metadata and tags on every row object, S3
|
||||
Select pushdown when the server answers it (probed once per process) and a local filter otherwise, bucket
|
||||
versioning for provenance, lifecycle expiry for ``runs/``, presigned URLs."""
|
||||
|
||||
def __init__(self, client: ObjectClient, bucket: str, prefix: str = "") -> None:
|
||||
self.client, self.bucket, self.prefix = client, bucket, prefix.strip("/")
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"minio://{self.bucket}" + (f"/{self.prefix}" if self.prefix else "")
|
||||
|
||||
def _k(self, key: str) -> str:
|
||||
return f"{self.prefix}/{key}" if self.prefix else key
|
||||
|
||||
def _get(self, key: str) -> tuple[bytes, str | None] | None:
|
||||
return self.client.get(self._k(key))
|
||||
|
||||
def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None:
|
||||
self.client.put(self._k(key), data, dict(labels or {}))
|
||||
|
||||
def _delete(self, key: str) -> None:
|
||||
self.client.delete(self._k(key))
|
||||
|
||||
def _keys(self, prefix: str) -> list[str]:
|
||||
cut = len(self.prefix) + 1 if self.prefix else 0
|
||||
return sorted(k[cut:] for k in self.client.keys(self._k(prefix)))
|
||||
|
||||
def _object_may_match(self, key: str, kind: str | None) -> bool:
|
||||
"""Filter by tag instead of reading: an object whose ``kind`` tag lacks ``kind`` holds no such row."""
|
||||
if kind is None:
|
||||
return True
|
||||
tagged = self.client.tags(self._k(key)).get("kind")
|
||||
return tagged is None or tagged == "*" or kind in tagged.split()
|
||||
|
||||
def select_supported(self) -> bool:
|
||||
"""Whether the server answers S3 Select over JSON Lines: one probe per bucket per process."""
|
||||
name = self.describe()
|
||||
if name not in _SELECT_PROBES:
|
||||
try:
|
||||
self.client.put(self._k(PROBE_KEY), b'{"probe":1}\n{"probe":2}\n', {})
|
||||
got = _parse_lines(self.client.select(self._k(PROBE_KEY), where_sql({"probe": 1})), PROBE_KEY)
|
||||
_SELECT_PROBES[name] = got == [{"probe": 1}]
|
||||
detail = "" if _SELECT_PROBES[name] else f"; the probe returned {got!r}"
|
||||
except Exception as exc: # noqa: BLE001 -- any failure means "not supported"; the fallback is exact
|
||||
_SELECT_PROBES[name], detail = False, f"; {type(exc).__name__}: {str(exc)[:200]}"
|
||||
if not _SELECT_PROBES[name]:
|
||||
log.warning("memory %s: the server does not answer S3 Select%s; filtering rows locally", name, detail)
|
||||
return _SELECT_PROBES[name]
|
||||
|
||||
def _select(self, key: str, where: Mapping[str, Any]) -> tuple[list[dict[str, Any]], str | None] | None:
|
||||
if not where or not self.select_supported():
|
||||
return super()._select(key, where)
|
||||
version = self.client.version(self._k(key)) # stat, then select: Select takes no version id
|
||||
if version is None and self.client.get(self._k(key)) is None:
|
||||
return None
|
||||
try:
|
||||
return _parse_lines(self.client.select(self._k(key), where_sql(where)), key), version
|
||||
except Exception as exc: # noqa: BLE001 -- one object's Select failing never loses rows
|
||||
log.warning("memory %s: S3 Select on %s failed (%s); filtering it locally", self.describe(), key, type(exc).__name__)
|
||||
return super()._select(key, where)
|
||||
|
||||
def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str:
|
||||
return self.client.presign("PUT", self._k(key), expires)
|
||||
|
||||
def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str:
|
||||
return self.client.presign("GET", self._k(key), expires)
|
||||
|
||||
def configure(self, runs_expire_days: int = 30) -> None:
|
||||
"""Bucket versioning on (provenance) and the lifecycle rule expiring ``runs/`` after ``runs_expire_days``;
|
||||
``repos/``, ``facts/`` and ``proposals/`` are kept."""
|
||||
if runs_expire_days < 1:
|
||||
raise ValueError("runs_expire_days must be at least 1")
|
||||
self.client.enable_versioning()
|
||||
self.client.expire(self._k("runs/"), runs_expire_days, "ousast-runs-expiry")
|
||||
|
||||
|
||||
class SdkClient:
|
||||
""":class:`ObjectClient` over ``minio.Minio``; imported lazily so the core install needs no SDK."""
|
||||
|
||||
def __init__(self, endpoint: str, access_key: str, secret_key: str, secure: bool, bucket: str) -> None:
|
||||
try:
|
||||
from minio import Minio
|
||||
except ImportError as exc:
|
||||
raise MemoryStoreError("OUSAST_MEMORY=minio://... needs the minio SDK: install openultrasast[minio]") from exc
|
||||
self.sdk, self.bucket = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure), bucket
|
||||
|
||||
def get(self, key: str) -> tuple[bytes, str | None] | None:
|
||||
from minio.error import S3Error
|
||||
|
||||
try:
|
||||
response = self.sdk.get_object(self.bucket, key)
|
||||
except S3Error as exc:
|
||||
if exc.code in ("NoSuchKey", "NoSuchObject"):
|
||||
return None
|
||||
raise
|
||||
try:
|
||||
return response.read(), response.headers.get("x-amz-version-id")
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def put(self, key: str, data: bytes, labels: Mapping[str, str]) -> None:
|
||||
import io
|
||||
|
||||
from minio.commonconfig import Tags
|
||||
|
||||
tags = Tags(for_object=True)
|
||||
for name, value in labels.items():
|
||||
tags[name] = value
|
||||
metadata: dict[str, str | list[str] | tuple[str]] = dict(labels)
|
||||
self.sdk.put_object(
|
||||
self.bucket, key, io.BytesIO(data), len(data), content_type="application/x-ndjson",
|
||||
metadata=metadata or None, tags=tags if labels else None,
|
||||
) # fmt: skip
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.sdk.remove_object(self.bucket, key)
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
return sorted(str(o.object_name) for o in self.sdk.list_objects(self.bucket, prefix=prefix, recursive=True))
|
||||
|
||||
def tags(self, key: str) -> dict[str, str]:
|
||||
return dict(self.sdk.get_object_tags(self.bucket, key) or {})
|
||||
|
||||
def version(self, key: str) -> str | None:
|
||||
from minio.error import S3Error
|
||||
|
||||
try:
|
||||
version = self.sdk.stat_object(self.bucket, key).version_id
|
||||
return None if version is None else str(version)
|
||||
except S3Error as exc:
|
||||
if exc.code in ("NoSuchKey", "NoSuchObject"):
|
||||
return None
|
||||
raise
|
||||
|
||||
def select(self, key: str, expression: str) -> bytes:
|
||||
from minio.select import JSONInputSerialization, JSONOutputSerialization, SelectRequest
|
||||
|
||||
request = SelectRequest(expression, JSONInputSerialization(json_type="LINES"), JSONOutputSerialization(), request_progress=False)
|
||||
with self.sdk.select_object_content(self.bucket, key, request) as result:
|
||||
return b"".join(result.stream())
|
||||
|
||||
def presign(self, method: str, key: str, expires: timedelta) -> str:
|
||||
if method == "PUT":
|
||||
return str(self.sdk.presigned_put_object(self.bucket, key, expires=expires))
|
||||
return str(self.sdk.presigned_get_object(self.bucket, key, expires=expires))
|
||||
|
||||
def expire(self, prefix: str, days: int, rule_id: str) -> None:
|
||||
from minio.commonconfig import ENABLED, Filter
|
||||
from minio.lifecycleconfig import Expiration, LifecycleConfig, Rule
|
||||
|
||||
rule = Rule(ENABLED, rule_filter=Filter(prefix=prefix), rule_id=rule_id, expiration=Expiration(days=days))
|
||||
self.sdk.set_bucket_lifecycle(self.bucket, LifecycleConfig([rule]))
|
||||
|
||||
def enable_versioning(self) -> None:
|
||||
from minio.commonconfig import ENABLED
|
||||
from minio.versioningconfig import VersioningConfig
|
||||
|
||||
self.sdk.set_bucket_versioning(self.bucket, VersioningConfig(ENABLED))
|
||||
|
||||
|
||||
def minio_settings(environ: Mapping[str, str] | None = None) -> dict[str, Any]:
|
||||
"""Endpoint, keys and TLS from ``.env``/the environment; a missing key is named, a value never is."""
|
||||
if environ is None:
|
||||
from ..config import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
environ = os.environ
|
||||
missing = [k for k in MINIO_KEYS if not environ.get(k)]
|
||||
if missing:
|
||||
raise MemoryStoreError(f"OUSAST_MEMORY=minio://... needs {', '.join(missing)} in .env or the environment")
|
||||
secure = (environ.get("MINIO_SECURE") or "true").strip().lower() not in ("0", "false", "no", "off")
|
||||
return {
|
||||
"endpoint": environ["MINIO_ENDPOINT"], "access_key": environ["MINIO_ACCESS_KEY"],
|
||||
"secret_key": environ["MINIO_SECRET_KEY"], "secure": secure,
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None) -> MemoryStore:
|
||||
"""The store ``spec`` (else ``OUSAST_MEMORY``, else ``file://<results_root()>/memory``) names."""
|
||||
from .reconciler import results_root
|
||||
|
||||
text = spec or (environ if environ is not None else os.environ).get("OUSAST_MEMORY") or ""
|
||||
if not text:
|
||||
return FileStore(results_root() / "memory")
|
||||
parts = urlsplit(text)
|
||||
if parts.scheme == "file" and parts.path:
|
||||
return FileStore(Path(parts.path))
|
||||
if parts.scheme == "minio" and parts.netloc:
|
||||
settings = minio_settings(environ)
|
||||
return MinioStore(SdkClient(bucket=parts.netloc, **settings), parts.netloc, parts.path)
|
||||
raise MemoryStoreError(f"OUSAST_MEMORY must be file:///<path> or minio://<bucket>[/<prefix>], got {text!r}")
|
||||
|
||||
|
||||
# --- ingest and seed -----------------------------------------------------------------------------------------------
|
||||
|
||||
REMEMBER_OUTPUT = "memory.jsonl"
|
||||
|
||||
|
||||
def read_memory(path: Path) -> list[dict[str, Any]]:
|
||||
"""A delivered ``memory.jsonl``: every row valid, else the error names the file and line."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if raw.strip():
|
||||
try:
|
||||
rows.append(validate_row(json.loads(raw), f"{path}:{number}"))
|
||||
except ValueError as exc:
|
||||
raise MemoryStoreError(f"{path}:{number}: {exc}") from exc
|
||||
return rows
|
||||
|
||||
|
||||
def ingest(run_dir: Path, store: MemoryStore | None = None) -> list[IngestResult]:
|
||||
"""Every delivered ``<task>/memory.jsonl`` of ``run_dir`` (and the ``facts.json`` beside it) into the store.
|
||||
A run without remember outputs is a no-op that says so; a malformed file fails before anything is written."""
|
||||
store = store or open_store()
|
||||
delivered = sorted(p for p in Path(run_dir).glob(f"*/{REMEMBER_OUTPUT}") if p.is_file())
|
||||
if not delivered:
|
||||
log.info("memory: %s has no remember outputs; nothing to ingest", run_dir)
|
||||
return []
|
||||
parsed = [(path, read_memory(path)) for path in delivered]
|
||||
results: list[IngestResult] = []
|
||||
for path, rows in parsed:
|
||||
facts = path.parent / "facts.json"
|
||||
results.append(store.ingest_rows(Path(run_dir).name, path.parent.name, rows, facts.read_bytes() if facts.is_file() else None))
|
||||
return results
|
||||
|
||||
|
||||
def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: Path | None = None) -> list[str]:
|
||||
"""Reuse stored facts for a Run's tasks whose ``openultrasast.io/memory-key`` matches (Req 6.1): each such task
|
||||
without a directory or a state gets ``facts.json`` and a ``done`` summary and is marked done under the run
|
||||
lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes."""
|
||||
from . import reconciler
|
||||
|
||||
run_spec, manifests = reconciler.load_run(Path(run_manifest))
|
||||
store = store or open_store()
|
||||
base = reconciler.run_dir(run_spec.metadata.name, results_root)
|
||||
state = (reconciler._read_json(base / "state.json") or {}).get("tasks") or {}
|
||||
seeded: list[str] = []
|
||||
for entry in run_spec.tasks:
|
||||
text = manifests.tasks[entry.task].metadata.annotations.get(MEMORY_KEY_ANNOTATION)
|
||||
if not text or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending":
|
||||
continue
|
||||
key = parse_memory_key(text)
|
||||
where = {"candidates_digest": key["candidates"], "image": key["image"]}
|
||||
for record in sorted(store.rows(key["repo"], key["pin"], "facts", where), key=lambda r: str(r.row["run"]), reverse=True):
|
||||
data = store.get_facts(str(record.row["sha256"]))
|
||||
if data is None:
|
||||
continue
|
||||
reused = {"sha256": record.row["sha256"], "from_run": record.row["run"], "image": key["image"]}
|
||||
(base / entry.name).mkdir(parents=True)
|
||||
(base / entry.name / "facts.json").write_bytes(data)
|
||||
summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None}
|
||||
(base / entry.name / "summary.json").write_text(json.dumps({**summary, "reused": reused}, indent=2, sort_keys=True) + "\n")
|
||||
reconciler.mark_done(run_spec.metadata.name, entry.name, reused=reused, results_root=results_root)
|
||||
seeded.append(entry.name)
|
||||
break
|
||||
return seeded
|
||||
|
||||
|
||||
def counts(results: Iterable[IngestResult]) -> dict[str, int]:
|
||||
total: dict[str, int] = {}
|
||||
for result in results:
|
||||
for kind, number in result.kinds.items():
|
||||
total[kind] = total.get(kind, 0) + number
|
||||
return dict(sorted(total.items()))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KINDS",
|
||||
"MEMORY_KEY_ANNOTATION",
|
||||
"POPULATION_ANNOTATION",
|
||||
"SPLIT_ANNOTATION",
|
||||
"FileStore",
|
||||
"IngestResult",
|
||||
"MemoryStore",
|
||||
"MemoryStoreError",
|
||||
"MinioStore",
|
||||
"ObjectClient",
|
||||
"Record",
|
||||
"SdkClient",
|
||||
"counts",
|
||||
"image_digest",
|
||||
"ingest",
|
||||
"memory_key",
|
||||
"open_store",
|
||||
"parse_memory_key",
|
||||
"repo_key",
|
||||
"row_id",
|
||||
"seed",
|
||||
"validate_row",
|
||||
"where_sql",
|
||||
]
|
||||
@@ -257,6 +257,18 @@ def _acquire_lock(base: Path) -> Path:
|
||||
return lock
|
||||
|
||||
|
||||
def mark_done(run_name: str, task: str, *, reused: Mapping[str, Any], results_root: Path | None = None) -> None:
|
||||
"""Set ``task`` done without running it (its outputs came from the memory store), under the run lock."""
|
||||
base = run_dir(run_name, results_root)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
lock = _acquire_lock(base)
|
||||
try:
|
||||
state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_name, "started": _now(), "tasks": {}})
|
||||
state.set(task, status="done", finished=_now(), usd=None, model=None, reused=dict(reused))
|
||||
finally:
|
||||
lock.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _await(ax: Ax, name: str, delivered: Callable[[], bool], start: Callable[[], str | None]) -> str | None:
|
||||
"""Resume ``name``, ``start`` it once Running (again after a re-resume), await delivery: None, else why it failed.
|
||||
``Failed`` is final only once a resume went through (ax sets it when a resume call fails); until then a
|
||||
@@ -462,4 +474,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""`remember` (harnessx-removal Requirement 6.1, design section 4): a case's delivered artifacts as memory rows.
|
||||
|
||||
No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the
|
||||
verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present,
|
||||
`alerts.jsonl`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`, `unit_cost` and `alert` (the
|
||||
fifth kind, `proposal_outcome`, comes from the loop) -- and passes `facts.json` through. Task code interprets the
|
||||
artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4).
|
||||
|
||||
Every row carries `id` (deterministic: :func:`..memory.row_id`), `kind`, `repo`, `pin`, `run`, `task`,
|
||||
`population`, `split` and `image` (the runner image digest the case's tasks ran on).
|
||||
|
||||
Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST_INPUT_FACTS`,
|
||||
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
|
||||
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS`; the repository and pin from the bound
|
||||
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
|
||||
`OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and
|
||||
`openultrasast.io/split` annotations). Exit 0 done, 2 failed.
|
||||
|
||||
On the host, :func:`remember_run` does the same over a recorded run directory (``ousast plane remember <run>``):
|
||||
per case without a delivered `memory.jsonl` it reads the artifacts and the rendered `task.yaml` beside them and
|
||||
ingests the rows directly, writing nothing into the run directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ..memory import REMEMBER_OUTPUT, IngestResult, MemoryStore, image_digest, ingest, repo_key, row_id, validate_row
|
||||
from .repo_facts import _digest
|
||||
|
||||
PASSES = ("a", "b", "c")
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Context:
|
||||
run: str
|
||||
task: str
|
||||
repo: str
|
||||
pin: str
|
||||
image: str
|
||||
population: str = UNKNOWN
|
||||
split: str = UNKNOWN
|
||||
|
||||
def row(self, kind: str, subject: str, **fields: Any) -> dict[str, Any]:
|
||||
base = {
|
||||
"id": row_id(kind, self.run, self.task, subject), "kind": kind, "repo": repo_key(self.repo), "pin": self.pin,
|
||||
"run": self.run, "task": self.task, "population": self.population, "split": self.split, "image": self.image,
|
||||
} # fmt: skip
|
||||
return {**base, **fields}
|
||||
|
||||
|
||||
def candidates_digest(facts: Mapping[str, Any]) -> str:
|
||||
"""`repo_facts._digest` of the candidate names `facts.json` was computed for (its `callers` keys)."""
|
||||
names = sorted({str(key).partition("::")[2] for key in (facts.get("callers") or {})})
|
||||
return _digest(names)
|
||||
|
||||
|
||||
def _final(row: Mapping[str, Any], disputed: set[str]) -> str:
|
||||
if row.get("agreed"):
|
||||
return "agreed"
|
||||
return "disputed" if str(row["candidate"]) in disputed else "rejected"
|
||||
|
||||
|
||||
def rows_for(
|
||||
ctx: Context,
|
||||
*,
|
||||
facts: bytes | None,
|
||||
passes: Mapping[str, Sequence[Mapping[str, Any]]],
|
||||
models: Mapping[str, str | None],
|
||||
agreed: Mapping[str, Any] | None,
|
||||
alerts: Sequence[Mapping[str, Any]] = (),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The memory rows of one case: facts, verdicts, unit costs, alerts."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
if facts is not None:
|
||||
loaded = json.loads(facts)
|
||||
sha = hashlib.sha256(facts).hexdigest()
|
||||
counts = loaded.get("counts") or {}
|
||||
files = int(counts.get("files", len(loaded.get("files") or {})))
|
||||
rows.append(ctx.row("facts", sha, sha256=sha, candidates_digest=candidates_digest(loaded), files=files))
|
||||
if agreed is not None:
|
||||
disputed = {str(d["candidate"]) for d in agreed.get("disputed") or []}
|
||||
for cand in agreed.get("candidates") or []:
|
||||
finding = cand.get("finding") or {}
|
||||
fields = {
|
||||
"candidate": cand["candidate"], "family": finding.get("family") or agreed.get("family"), "site": cand.get("site"),
|
||||
"passes": {p: cand.get(p) for p in PASSES}, "final": _final(cand, disputed), "tiebreak": cand.get("c") is not None,
|
||||
"site_match": cand.get("site_match"), "usd": cand.get("usd"), "turns": cand.get("turns"),
|
||||
} # fmt: skip
|
||||
rows.append(ctx.row("verdict", str(cand["candidate"]), **fields))
|
||||
for label in PASSES:
|
||||
for unit in passes.get(label) or []:
|
||||
usage = dict(unit.get("usage") or {})
|
||||
fields = {
|
||||
"pass": label, "path": unit.get("path"), "usd": unit.get("usd"), "calls": usage.get("calls"), "usage": usage,
|
||||
"turns": unit.get("turns"), "model": models.get(label),
|
||||
} # fmt: skip
|
||||
rows.append(ctx.row("unit_cost", f"{label}:{unit.get('path')}", **fields))
|
||||
for alert in alerts:
|
||||
fields = {k: alert.get(k) for k in ("rule_id", "rule_status", "path", "line", "function", "pin_role")}
|
||||
rows.append(ctx.row("alert", f"{fields['rule_id']}:{fields['path']}:{fields['line']}", **fields))
|
||||
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
|
||||
|
||||
|
||||
def _jsonl(path: Path | None) -> list[dict[str, Any]]:
|
||||
if path is None or not path.is_file():
|
||||
return []
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _json(path: Path | None) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path is not None and path.is_file() else None
|
||||
|
||||
|
||||
def _model(summary: object) -> str | None:
|
||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||
|
||||
|
||||
def context_from(
|
||||
task_doc: Mapping[str, Any], workspaces: Sequence[Mapping[str, Any]], *, run: str, task: str, population: str, split: str
|
||||
) -> Context:
|
||||
"""Repository, pin and image from a rendered Task and its Workspaces: the pin named in `OUSAST_GIT_PINS`, the
|
||||
repository of that Workspace's git entry, the Task's image."""
|
||||
spec = task_doc.get("spec") or {}
|
||||
env = {str(e.get("name")): str(e.get("value")) for e in spec.get("env") or []}
|
||||
pins: dict[str, str] = json.loads(env.get("OUSAST_GIT_PINS") or "{}")
|
||||
if len(pins) != 1:
|
||||
raise ValueError(f"{task}: expected one pinned repository in OUSAST_GIT_PINS, got {sorted(pins)}")
|
||||
((ref, pin),) = pins.items()
|
||||
ws_name, _, git_name = ref.partition("/")
|
||||
repos = [
|
||||
str(g["repo"])
|
||||
for w in workspaces if (w.get("metadata") or {}).get("name") == ws_name
|
||||
for g in (w.get("spec") or {}).get("git") or [] if g.get("name") == git_name
|
||||
] # fmt: skip
|
||||
if not repos:
|
||||
raise ValueError(f"{task}: no bound Workspace git entry {ref}")
|
||||
image = str(spec.get("image") or "")
|
||||
if not image:
|
||||
raise ValueError(f"{task}: the Task names no image")
|
||||
return Context(run, task, repos[0], pin, image_digest(image), population or UNKNOWN, split or UNKNOWN)
|
||||
|
||||
|
||||
def _rendered(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
docs = [d for d in yaml.safe_load_all(path.read_text(encoding="utf-8")) if isinstance(d, dict)]
|
||||
tasks = [d for d in docs if d.get("kind") == "Task"]
|
||||
if not tasks:
|
||||
raise ValueError(f"{path}: no rendered Task")
|
||||
return tasks[-1], [d for d in docs if d.get("kind") == "Workspace"]
|
||||
|
||||
|
||||
def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[list[dict[str, Any]], bytes | None]:
|
||||
"""The rows of one case of a recorded run directory, from its artifacts and the rendered `task.yaml`."""
|
||||
facts_dir = base / f"{case}-facts"
|
||||
candidates = [base / f"{case}-{step}" / "task.yaml" for step in ("facts", "va", "vb", "agree")]
|
||||
rendered = next((path for path in candidates if path.is_file()), None)
|
||||
if rendered is None:
|
||||
raise ValueError(f"{base}: case {case} has no rendered task.yaml")
|
||||
task_doc, workspaces = _rendered(rendered)
|
||||
ctx = context_from(task_doc, workspaces, run=base.name, task=f"{case}-remember", population=population, split=split)
|
||||
reused = (_json(facts_dir / "summary.json") or {}).get("reused")
|
||||
if isinstance(reused, dict) and reused.get("image"):
|
||||
ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, image_digest(str(reused["image"])), ctx.population, ctx.split)
|
||||
facts = (facts_dir / "facts.json").read_bytes() if (facts_dir / "facts.json").is_file() else None
|
||||
final = base / f"{case}-final" / "agreed.json"
|
||||
agreed = _json(final if final.is_file() else base / f"{case}-agree" / "agreed.json")
|
||||
passes = {p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES}
|
||||
models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES}
|
||||
alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl")
|
||||
return rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts), facts
|
||||
|
||||
|
||||
def remember_run(run_dir: Path, store: MemoryStore, *, population: str = UNKNOWN, split: str = UNKNOWN) -> list[IngestResult]:
|
||||
"""Ingest a run: delivered `memory.jsonl` files as they are, and every other case (a `<case>-facts` task)
|
||||
computed here from its artifacts."""
|
||||
base = Path(run_dir)
|
||||
if not base.is_dir():
|
||||
raise ValueError(f"no run directory at {base}")
|
||||
results = ingest(base, store)
|
||||
delivered = {r.task for r in results}
|
||||
for facts_dir in sorted(base.glob("*-facts")):
|
||||
case = facts_dir.name.removesuffix("-facts")
|
||||
if f"{case}-remember" in delivered or (base / f"{case}-remember" / REMEMBER_OUTPUT).is_file():
|
||||
continue
|
||||
rows, facts = case_rows(base, case, population=population, split=split)
|
||||
results.append(store.ingest_rows(base.name, f"{case}-remember", rows, facts))
|
||||
return results
|
||||
|
||||
|
||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
|
||||
def _units(path: Path | None) -> list[dict[str, Any]]:
|
||||
return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path)
|
||||
|
||||
|
||||
def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
env = environ if environ is not None else os.environ
|
||||
output_dir = _path(env, "OUSAST_OUTPUT_DIR")
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
try:
|
||||
if output_dir is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_doc = yaml.safe_load(env.get("AX_TASK_YAML") or "{}") or {}
|
||||
task_doc.setdefault("spec", {})["env"] = [{"name": "OUSAST_GIT_PINS", "value": env.get("OUSAST_GIT_PINS") or "{}"}]
|
||||
workspaces = [d for d in yaml.safe_load_all(env.get("AX_WORKSPACES_YAML") or "") if isinstance(d, dict)]
|
||||
ctx = context_from(
|
||||
task_doc, workspaces, run=env.get("OUSAST_RUN") or UNKNOWN, task=env.get("OUSAST_TASK") or "remember",
|
||||
population=env.get("OUSAST_POPULATION") or UNKNOWN, split=env.get("OUSAST_SPLIT") or UNKNOWN,
|
||||
) # fmt: skip
|
||||
facts_path = _path(env, "OUSAST_INPUT_FACTS")
|
||||
facts = facts_path.read_bytes() if facts_path is not None and facts_path.is_file() else None
|
||||
passes = {p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES}
|
||||
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
|
||||
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
|
||||
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")))
|
||||
(output_dir / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
|
||||
if facts is not None:
|
||||
(output_dir / "facts.json").write_bytes(facts)
|
||||
summary.update(status="done", units_done=1, rows=len(rows))
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
if output_dir is not None:
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: summary.get(k) for k in ("status", "rows")}), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["Context", "candidates_digest", "case_rows", "context_from", "main", "remember_run", "rows_for"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"family": "untrusted_destination",
|
||||
"candidates": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"a": true,
|
||||
"b": true,
|
||||
"agreed": true,
|
||||
"disputed": false,
|
||||
"site_match": true,
|
||||
"usd": 0.022175,
|
||||
"turns": 10,
|
||||
"finding": {
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"agreed": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c",
|
||||
"passes": 2
|
||||
}
|
||||
],
|
||||
"disputed": [],
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[]
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": null,
|
||||
"pass": null,
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"callers": {
|
||||
"lmdeploy/vl/media/connection.py::_load_http_url": []
|
||||
},
|
||||
"counts": {
|
||||
"callers": 0,
|
||||
"candidates": 1,
|
||||
"files": 1,
|
||||
"functions": 4809
|
||||
},
|
||||
"files": {
|
||||
"lmdeploy/vl/media/connection.py": {
|
||||
"functions": [
|
||||
"_is_safe_url",
|
||||
"_load_http_url",
|
||||
"_load_data_url",
|
||||
"_load_file_url",
|
||||
"load_from_url"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"calls": 0,
|
||||
"model": null,
|
||||
"status": "done",
|
||||
"units_done": 608,
|
||||
"units_total": 608,
|
||||
"usage": {},
|
||||
"usd": null
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Workspace
|
||||
metadata:
|
||||
name: lmdeploy-media-url-ssrf-vulnerable
|
||||
atespace: default
|
||||
spec:
|
||||
git:
|
||||
- name: repo
|
||||
repo: https://example.com/reserved/lmdeploy
|
||||
dir: repo
|
||||
depth: 1
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: validation-46-lmdeploy-media-url-ssrf-facts
|
||||
atespace: default
|
||||
spec:
|
||||
command:
|
||||
- repo-facts
|
||||
env:
|
||||
- name: OUSAST_RUN
|
||||
value: validation-46
|
||||
- name: OUSAST_TASK
|
||||
value: lmdeploy-media-url-ssrf-facts
|
||||
- name: OUSAST_GIT_PINS
|
||||
value: '{"lmdeploy-media-url-ssrf-vulnerable/repo": "71648d239c978452720a6d6692fe914efc8a7b94"}'
|
||||
image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1280Mi
|
||||
workspaces:
|
||||
- name: lmdeploy-media-url-ssrf-vulnerable
|
||||
path: /workspace/case
|
||||
- name: lmdeploy-media-url-ssrf-inputs
|
||||
path: /workspace/inputs
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"family": "untrusted_destination",
|
||||
"candidates": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"a": true,
|
||||
"b": true,
|
||||
"c": null,
|
||||
"agreed": true,
|
||||
"disputed": false,
|
||||
"site_match": true,
|
||||
"usd": 0.022175,
|
||||
"turns": 10,
|
||||
"finding": {
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"agreed": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c",
|
||||
"passes": 2
|
||||
}
|
||||
],
|
||||
"disputed": [],
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0,
|
||||
"tiebreak": {
|
||||
"asked": 0,
|
||||
"flipped_to_agreed": 0,
|
||||
"flipped_to_rejected": 0,
|
||||
"rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[]
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": null,
|
||||
"pass": null,
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0,
|
||||
"tiebreak": {
|
||||
"asked": 0,
|
||||
"flipped_to_agreed": 0,
|
||||
"flipped_to_rejected": 0,
|
||||
"rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)"
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0.012376,
|
||||
"calls": 7,
|
||||
"usage": {
|
||||
"prompt_tokens": 68714,
|
||||
"prompt_cache_hit_tokens": 48512,
|
||||
"completion_tokens": 2127
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "a",
|
||||
"family": "untrusted_destination"
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "a", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF guard bypass via HTTP redirect in _load_http_url", "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"}], "turns": 5, "usage": {"prompt_tokens": 68714, "prompt_cache_hit_tokens": 48512, "completion_tokens": 2127, "calls": 7}, "usd": 0.01237569}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0.009799,
|
||||
"calls": 7,
|
||||
"usage": {
|
||||
"prompt_tokens": 61163,
|
||||
"prompt_cache_hit_tokens": 46464,
|
||||
"completion_tokens": 2031
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "b",
|
||||
"family": "untrusted_destination"
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "b", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF via unvalidated HTTP redirects in media URL fetch", "witness": "Source: an HTTP client submits a chat request whose multimodal content contains an attacker-controlled URL, e.g. {'type'"}], "turns": 5, "usage": {"prompt_tokens": 61163, "prompt_cache_hit_tokens": 46464, "completion_tokens": 2031, "calls": 7}, "usd": 0.00979898}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 0,
|
||||
"units_total": 0,
|
||||
"usd": 0.0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "c",
|
||||
"family": "untrusted_destination",
|
||||
"only": 0
|
||||
}
|
||||
@@ -19,6 +19,8 @@ from openultrasast.plane import reconciler
|
||||
from openultrasast.plane.budget import prices_from
|
||||
from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates
|
||||
from openultrasast.plane.manifests import load_manifests
|
||||
from openultrasast.plane.memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, parse_memory_key
|
||||
from openultrasast.plane.tasks.repo_facts import _digest
|
||||
|
||||
PLANE = Path("plane")
|
||||
RUN = PLANE / "runs" / "validation-46.yaml"
|
||||
@@ -160,7 +162,8 @@ def test_committed_manifests_match_the_generator() -> None:
|
||||
POPULATION,
|
||||
REPOS,
|
||||
)
|
||||
for relative, text in render(cases, templates, "validation-46", _committed_command()).items():
|
||||
rendered = render(cases, templates, "validation-46", _committed_command(), population=POPULATION.stem, split="validation")
|
||||
for relative, text in rendered.items():
|
||||
assert (PLANE / relative).read_text() == text, relative
|
||||
|
||||
|
||||
@@ -216,6 +219,12 @@ def test_generator_is_deterministic_and_follows_the_reference(tmp_path: Path) ->
|
||||
assert inputs["triage.json"]["candidates"] == [["a.py", "helper", 20], ["a.py", "run", 9]]
|
||||
assert run.task("alpha-va").budget is not None and run.task("alpha-va").budget.usd == 0.38 # 1.5 * 0.5 / 2, cent up
|
||||
assert yaml.safe_load(first[0].read_text())["metadata"]["annotations"] == {"openultrasast.io/git-commits": f"repo={PIN}"}
|
||||
# memory (harnessx-removal §4): the facts Task names its reuse key, the Run its population and split
|
||||
key = parse_memory_key(manifests.tasks["repo-facts-alpha"].metadata.annotations[MEMORY_KEY_ANNOTATION])
|
||||
image = manifests.tasks["repo-facts-alpha"].image or ""
|
||||
assert key == {"repo": "example.com/alpha", "pin": PIN, "candidates": _digest(["run"]), "image": image.rpartition("@")[2]}
|
||||
assert all(MEMORY_KEY_ANNOTATION not in t.metadata.annotations for n, t in manifests.tasks.items() if not n.startswith("repo-facts-"))
|
||||
assert dict(run.metadata.annotations) == {POPULATION_ANNOTATION: "population", SPLIT_ANNOTATION: "set"}
|
||||
|
||||
|
||||
def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path) -> None:
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
"""The plane's memory store (harnessx-removal Req 6.1/6.4, design §4): one contract over every backend, the S3
|
||||
Select probe and its fallback on a fake object client, backend selection, fact reuse by ``seed``, disk refusal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.plane import memory, reconciler
|
||||
from openultrasast.plane.memory import (
|
||||
MINIO_KEYS,
|
||||
FileStore,
|
||||
MemoryStore,
|
||||
MemoryStoreError,
|
||||
MinioStore,
|
||||
SdkClient,
|
||||
memory_key,
|
||||
minio_settings,
|
||||
open_store,
|
||||
row_id,
|
||||
seed,
|
||||
where_sql,
|
||||
)
|
||||
from openultrasast.plane.tasks.repo_facts import _digest
|
||||
|
||||
PIN = "0123456789abcdef0123456789abcdef01234567"
|
||||
OTHER_PIN = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
IMAGE = "sha256:" + "a" * 64
|
||||
FACTS = json.dumps({"callers": {"a.py::run": []}, "counts": {"files": 1}, "files": {"a.py": {"functions": ["run"]}}}).encode()
|
||||
MINIO_SKIP = (
|
||||
"MinIO contract tests need OUSAST_MEMORY_TEST_MINIO=1 plus MINIO_ENDPOINT, MINIO_ACCESS_KEY and MINIO_SECRET_KEY "
|
||||
"exported (optional: MINIO_SECURE, OUSAST_MEMORY_TEST_BUCKET, default ousast-memory-test) and the minio extra installed"
|
||||
)
|
||||
|
||||
|
||||
def row(kind: str, subject: str, *, pin: str = PIN, run: str = "r1", repo: str = "https://github.com/o/r", **fields: Any) -> dict[str, Any]:
|
||||
task = "c-remember"
|
||||
base = {"id": row_id(kind, run, task, subject), "kind": kind, "repo": memory.repo_key(repo), "pin": pin, "run": run, "task": task}
|
||||
return {**base, "population": "population-v2", "split": "validation", "image": IMAGE, **fields}
|
||||
|
||||
|
||||
def verdict(candidate: str, final: str, **fields: Any) -> dict[str, Any]:
|
||||
return row("verdict", candidate, candidate=candidate, family="injection", final=final, tiebreak=False, **fields)
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
"""An in-memory :class:`memory.ObjectClient`: versions per write, tags per object; ``select`` works, raises or
|
||||
answers wrongly, evaluating the equality clauses :func:`where_sql` writes."""
|
||||
|
||||
def __init__(self, select: str = "works") -> None:
|
||||
self.mode, self.objects, self.versions = select, {}, {}
|
||||
self.selects: list[str] = []
|
||||
self.gets: list[str] = []
|
||||
self.calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def get(self, key: str) -> tuple[bytes, str | None] | None:
|
||||
self.gets.append(key)
|
||||
found = self.objects.get(key)
|
||||
return None if found is None else (found[0], f"v{self.versions[key]}")
|
||||
|
||||
def put(self, key: str, data: bytes, labels: dict[str, str]) -> None:
|
||||
self.versions[key] = self.versions.get(key, 0) + 1
|
||||
self.objects[key] = (data, dict(labels))
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.objects.pop(key, None)
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
return sorted(k for k in self.objects if k.startswith(prefix))
|
||||
|
||||
def tags(self, key: str) -> dict[str, str]:
|
||||
return dict(self.objects[key][1])
|
||||
|
||||
def version(self, key: str) -> str | None:
|
||||
return f"v{self.versions[key]}" if key in self.objects else None
|
||||
|
||||
def select(self, key: str, expression: str) -> bytes:
|
||||
self.selects.append(expression)
|
||||
if self.mode == "raises":
|
||||
raise RuntimeError("NotImplemented: SelectObjectContent is not supported")
|
||||
if self.mode == "wrong":
|
||||
return b""
|
||||
clauses = re.findall(r"s\.\"(\w+)\" = ('(?:[^']|'')*'|true|false|-?[\d.]+)", expression)
|
||||
want = {n: v[1:-1].replace("''", "'") if v.startswith("'") else json.loads(v) for n, v in clauses}
|
||||
rows = [json.loads(line) for line in self.objects[key][0].splitlines() if line.strip()]
|
||||
return b"".join(json.dumps(r).encode() + b"\n" for r in rows if all(r.get(k) == v for k, v in want.items()))
|
||||
|
||||
def presign(self, method: str, key: str, expires: timedelta) -> str:
|
||||
return f"https://minio.test/{key}?method={method}&X-Amz-Expires={int(expires.total_seconds())}"
|
||||
|
||||
def expire(self, prefix: str, days: int, rule_id: str) -> None:
|
||||
self.calls.append(("expire", prefix, days, rule_id))
|
||||
|
||||
def enable_versioning(self) -> None:
|
||||
self.calls.append(("versioning",))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh_probes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(memory, "_SELECT_PROBES", {})
|
||||
|
||||
|
||||
def _real_minio() -> Iterator[MemoryStore]:
|
||||
if os.environ.get("OUSAST_MEMORY_TEST_MINIO") != "1" or any(not os.environ.get(k) for k in MINIO_KEYS):
|
||||
pytest.skip(MINIO_SKIP)
|
||||
pytest.importorskip("minio", reason=MINIO_SKIP)
|
||||
bucket = os.environ.get("OUSAST_MEMORY_TEST_BUCKET") or "ousast-memory-test"
|
||||
client = SdkClient(bucket=bucket, **minio_settings(os.environ))
|
||||
if not client.sdk.bucket_exists(bucket):
|
||||
client.sdk.make_bucket(bucket)
|
||||
store = MinioStore(client, bucket, f"contract-{uuid.uuid4().hex[:12]}")
|
||||
client.enable_versioning()
|
||||
yield store
|
||||
for key in client.keys(store.prefix + "/"):
|
||||
client.delete(key)
|
||||
|
||||
|
||||
@pytest.fixture(params=["file", "fake-minio-select", "fake-minio-fallback", "minio"])
|
||||
def store(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[MemoryStore]:
|
||||
if request.param == "file":
|
||||
yield FileStore(tmp_path / "memory")
|
||||
elif request.param == "minio":
|
||||
yield from _real_minio()
|
||||
else:
|
||||
yield MinioStore(FakeObjects("works" if request.param.endswith("select") else "raises"), "bucket", "p")
|
||||
|
||||
|
||||
# --- the contract --------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ingest_is_idempotent_by_the_index(store: MemoryStore) -> None:
|
||||
rows = [verdict("a.py::run", "agreed"), verdict("a.py::other", "rejected")]
|
||||
first = store.ingest_rows("r1", "c-remember", rows, FACTS)
|
||||
again = store.ingest_rows("r1", "c-remember", rows, FACTS)
|
||||
assert (first.skipped, again.skipped, first.kinds) == (False, True, {"verdict": 2})
|
||||
assert [(e["run"], e["task"], e["rows"]) for e in store.index()] == [("r1", "c-remember", 2)]
|
||||
assert len(store.rows(kind="verdict")) == 2
|
||||
|
||||
|
||||
def test_identical_facts_are_stored_once_and_read_back_by_content(store: MemoryStore) -> None:
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
store.ingest_rows("r1", "c-remember", [row("facts", sha, sha256=sha)], FACTS)
|
||||
store.ingest_rows("r2", "c-remember", [row("facts", sha, run="r2", sha256=sha)], FACTS)
|
||||
assert store._keys("facts/") == [f"facts/{sha}.json"]
|
||||
assert store.get_facts(sha) == FACTS and store.get_facts("0" * 64) is None
|
||||
|
||||
|
||||
def test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version(store: MemoryStore) -> None:
|
||||
store.put_rows([
|
||||
verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), verdict("b.py::y", "agreed", pin=OTHER_PIN),
|
||||
verdict("c.py::z", "agreed", repo="https://github.com/o/other.git"), row("unit_cost", "a:a.py", **{"pass": "a", "usd": 0.5}),
|
||||
]) # fmt: skip
|
||||
assert len(store.rows()) == 5
|
||||
assert {r.row["candidate"] for r in store.rows("https://github.com/o/r", kind="verdict")} == {"a.py::run", "a.py::x", "b.py::y"}
|
||||
assert [r.row["candidate"] for r in store.rows("github.com/o/r", PIN, "verdict", {"final": "disputed"})] == ["a.py::x"]
|
||||
assert [r.row["kind"] for r in store.rows(where={"usd": 0.5})] == ["unit_cost"]
|
||||
assert store.rows(where={"final": "no such"}) == []
|
||||
record = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0]
|
||||
assert record.key == f"repos/github.com__o__r/{PIN}.jsonl" and record.version
|
||||
store.put_row(verdict("a.py::new", "agreed"))
|
||||
later = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0]
|
||||
assert later.row == record.row and later.version != record.version, "a rewrite is a new version: provenance pins the old"
|
||||
|
||||
|
||||
def test_a_repeated_id_replaces_the_stored_row(store: MemoryStore) -> None:
|
||||
store.put_row(verdict("a.py::run", "disputed"))
|
||||
store.put_row(verdict("a.py::run", "agreed"))
|
||||
assert [r.row["final"] for r in store.rows(kind="verdict")] == ["agreed"]
|
||||
|
||||
|
||||
def test_a_malformed_row_fails_naming_it_and_writes_nothing(store: MemoryStore) -> None:
|
||||
bad = {**verdict("a.py::run", "agreed"), "pin": "main"}
|
||||
with pytest.raises(MemoryStoreError, match=r"r1/c-remember row 2: pin 'main'"):
|
||||
store.ingest_rows("r1", "c-remember", [verdict("a.py::ok", "agreed"), bad])
|
||||
with pytest.raises(MemoryStoreError, match="unknown kind"):
|
||||
store.put_row({**verdict("a.py::run", "agreed"), "kind": "vibes"})
|
||||
assert store.index() == [] and store.rows() == []
|
||||
|
||||
|
||||
def test_facts_that_no_longer_match_their_hash_are_dropped(store: MemoryStore, caplog: pytest.LogCaptureFixture) -> None:
|
||||
sha = store.put_facts(FACTS)
|
||||
store._put(f"facts/{sha}.json", b"tampered")
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
assert store.get_facts(sha) is None
|
||||
assert "no longer matches" in caplog.text and store._get(f"facts/{sha}.json") is None
|
||||
|
||||
|
||||
def test_presigned_urls_name_the_object(store: MemoryStore) -> None:
|
||||
for url in (store.presign_put("runs/r1/x/units.jsonl"), store.presign_get("runs/r1/x/units.jsonl")):
|
||||
assert url.startswith(("file://", "http://", "https://")) and "runs/r1/x/units.jsonl" in url
|
||||
|
||||
|
||||
# --- MinIO specifics on the fake client ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _filled(client: FakeObjects) -> MinioStore:
|
||||
store = MinioStore(client, "bucket")
|
||||
store.put_rows([verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), row("unit_cost", "a:a.py", **{"pass": "a"})])
|
||||
return store
|
||||
|
||||
|
||||
def test_select_is_pushed_down_when_the_probe_answers(caplog: pytest.LogCaptureFixture) -> None:
|
||||
client = FakeObjects("works")
|
||||
store = _filled(client)
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
got = [r.row["candidate"] for r in store.rows(kind="verdict", where={"final": "disputed"})]
|
||||
assert got == ["a.py::x"] and store.select_supported() and "locally" not in caplog.text
|
||||
assert client.selects[0] == 'SELECT * FROM S3Object s WHERE s."probe" = 1'
|
||||
assert client.selects[1:] == ["SELECT * FROM S3Object s WHERE s.\"final\" = 'disputed' AND s.\"kind\" = 'verdict'"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["raises", "wrong"])
|
||||
def test_without_select_the_same_rows_come_from_a_local_filter_logged_once(
|
||||
mode: str, caplog: pytest.LogCaptureFixture, tmp_path: Path
|
||||
) -> None:
|
||||
client = FakeObjects(mode)
|
||||
store, reference = _filled(client), FileStore(tmp_path / "m")
|
||||
reference.put_rows([r.row for r in store.rows()])
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
for where in ({"final": "disputed"}, {"final": "agreed"}, {"pass": "a"}):
|
||||
assert [r.row for r in store.rows(where=where)] == [r.row for r in reference.rows(where=where)] != []
|
||||
assert not store.select_supported() and len(client.selects) == 1, "one probe per process"
|
||||
assert caplog.text.count("does not answer S3 Select") == 1
|
||||
|
||||
|
||||
def test_a_failing_select_on_one_object_falls_back_for_that_object(caplog: pytest.LogCaptureFixture) -> None:
|
||||
client = FakeObjects("works")
|
||||
store = _filled(client)
|
||||
assert store.select_supported()
|
||||
client.mode = "raises"
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
assert [r.row["candidate"] for r in store.rows(where={"final": "agreed"})] == ["a.py::run"]
|
||||
assert "filtering it locally" in caplog.text
|
||||
|
||||
|
||||
def test_row_objects_carry_tags_and_a_kind_tag_skips_objects_without_that_kind() -> None:
|
||||
client = FakeObjects("works")
|
||||
store = MinioStore(client, "bucket")
|
||||
store.put_rows([verdict("a.py::run", "agreed"), row("unit_cost", "a:b.py", pin=OTHER_PIN, **{"pass": "a"})])
|
||||
tags = client.tags(f"repos/github.com__o__r/{PIN}.jsonl")
|
||||
assert tags == {"repo": "github.com/o/r", "pin": PIN, "kind": "verdict", "family": "injection", "run": "r1",
|
||||
"population": "population-v2", "split": "validation"} # fmt: skip
|
||||
client.gets.clear()
|
||||
assert [r.row["kind"] for r in store.rows(kind="unit_cost")] == ["unit_cost"]
|
||||
assert f"repos/github.com__o__r/{PIN}.jsonl" not in client.gets, "an object tagged without the kind is not read"
|
||||
|
||||
|
||||
def test_configure_turns_versioning_on_and_expires_runs_only() -> None:
|
||||
client = FakeObjects()
|
||||
MinioStore(client, "bucket", "memory").configure(14)
|
||||
assert client.calls == [("versioning",), ("expire", "memory/runs/", 14, "ousast-runs-expiry")]
|
||||
with pytest.raises(ValueError):
|
||||
MinioStore(client, "bucket").configure(0)
|
||||
|
||||
|
||||
def test_where_sql_quotes_values_and_refuses_odd_fields() -> None:
|
||||
assert where_sql({}) == "SELECT * FROM S3Object s"
|
||||
assert where_sql({"a": "it's", "b": True, "c": None, "d": 2}) == (
|
||||
'SELECT * FROM S3Object s WHERE s."a" = \'it\'\'s\' AND s."b" = true AND s."c" IS NULL AND s."d" = 2'
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
where_sql({"a; drop": 1})
|
||||
|
||||
|
||||
# --- selection and settings ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_open_store_selects_the_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.delenv("OUSAST_MEMORY", raising=False)
|
||||
default = open_store()
|
||||
assert isinstance(default, FileStore) and default.root == tmp_path / "plane" / "memory"
|
||||
chosen = open_store(environ={"OUSAST_MEMORY": f"file://{tmp_path}/m"})
|
||||
assert isinstance(chosen, FileStore) and chosen.root == tmp_path / "m"
|
||||
with pytest.raises(MemoryStoreError, match="file:///<path> or minio://<bucket>"):
|
||||
open_store("s3://bucket")
|
||||
|
||||
|
||||
def test_minio_settings_name_missing_keys_never_values() -> None:
|
||||
with pytest.raises(MemoryStoreError, match="MINIO_ACCESS_KEY, MINIO_SECRET_KEY") as caught:
|
||||
open_store(environ={"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "minio.example:9000"})
|
||||
assert "minio.example" not in str(caught.value)
|
||||
env = {"MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "sk-value", "MINIO_SECURE": "false"}
|
||||
assert minio_settings(env) == {"endpoint": "h:9000", "access_key": "ak", "secret_key": "sk-value", "secure": False}
|
||||
assert minio_settings({**env, "MINIO_SECURE": ""})["secure"] is True
|
||||
|
||||
|
||||
def test_minio_without_the_sdk_names_the_extra() -> None:
|
||||
try:
|
||||
import minio # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
pytest.skip("the minio SDK is installed here")
|
||||
env = {"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "secret-xyz"}
|
||||
with pytest.raises(MemoryStoreError, match=r"install openultrasast\[minio\]") as caught:
|
||||
open_store(environ=env)
|
||||
assert "secret-xyz" not in str(caught.value)
|
||||
|
||||
|
||||
# --- disk refusal (FileStore) --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_store_refuses_below_one_gib_free_naming_its_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
root = tmp_path / "memory"
|
||||
monkeypatch.setattr(memory.shutil, "disk_usage", lambda path: shutil._ntuple_diskusage(100 << 30, 99 << 30, 512 << 20)) # type: ignore[attr-defined]
|
||||
with pytest.raises(MemoryStoreError, match=rf"refusing to write the memory store at {re.escape(str(root))}: 0.50 GiB free"):
|
||||
FileStore(root).ingest_rows("r1", "c-remember", [verdict("a.py::run", "agreed")], FACTS)
|
||||
assert not root.exists()
|
||||
|
||||
|
||||
# --- fact reuse ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _plane(root: Path, key: str, run: str = "r2") -> Path:
|
||||
(root / "runs").mkdir(parents=True)
|
||||
(root / "tasks").mkdir()
|
||||
(root / "tasks" / "t.yaml").write_text(
|
||||
"apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: repo-facts-c\n annotations:\n"
|
||||
f" {memory.MEMORY_KEY_ANNOTATION}: '{key}'\nspec:\n image: reg/img@{IMAGE}\n command: [repo-facts]\n"
|
||||
)
|
||||
(root / "runs" / f"{run}.yaml").write_text(
|
||||
f"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata:\n name: {run}\nspec:\n tasks:\n"
|
||||
" - name: c-facts\n task: repo-facts-c\n outputs: [facts.json]\n"
|
||||
)
|
||||
return root / "runs" / f"{run}.yaml"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stored(tmp_path: Path) -> FileStore:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
facts = row("facts", sha, sha256=sha, candidates_digest=_digest(["run"]), files=1)
|
||||
store.ingest_rows("r1", "c-remember", [facts], FACTS)
|
||||
return store
|
||||
|
||||
|
||||
def test_seed_reuses_facts_for_an_unchanged_key_and_marks_the_task_done(stored: FileStore, tmp_path: Path) -> None:
|
||||
manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}"))
|
||||
results = tmp_path / "results"
|
||||
assert seed(manifest, stored, results_root=results) == ["c-facts"]
|
||||
base = results / "r2"
|
||||
assert (base / "c-facts" / "facts.json").read_bytes() == FACTS
|
||||
summary = json.loads((base / "c-facts" / "summary.json").read_text())
|
||||
reused = {"sha256": hashlib.sha256(FACTS).hexdigest(), "from_run": "r1", "image": IMAGE}
|
||||
assert reconciler.outcome_of(summary) == ("done", "") and summary["reused"] == reused
|
||||
state = json.loads((base / "state.json").read_text())
|
||||
assert state["tasks"]["c-facts"]["status"] == "done" and state["tasks"]["c-facts"]["reused"] == reused
|
||||
assert not (base / "lock").exists()
|
||||
assert seed(manifest, stored, results_root=results) == [], "a seeded task is never seeded again"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change", [{"pin": OTHER_PIN}, {"candidates": _digest(["run", "helper"])}, {"image": "reg/img@sha256:" + "b" * 64}]
|
||||
)
|
||||
def test_seed_recomputes_on_a_changed_pin_candidate_set_or_image(stored: FileStore, tmp_path: Path, change: dict[str, str]) -> None:
|
||||
key = {"repo": "https://github.com/o/r", "pin": PIN, "candidates": _digest(["run"]), "image": f"reg/img@{IMAGE}", **change}
|
||||
manifest = _plane(tmp_path / "plane", memory_key(**key))
|
||||
assert seed(manifest, stored, results_root=tmp_path / "results") == []
|
||||
assert not (tmp_path / "results" / "r2").exists()
|
||||
|
||||
|
||||
def test_seed_never_overwrites_a_task_directory_and_drops_tampered_facts(stored: FileStore, tmp_path: Path) -> None:
|
||||
manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}"))
|
||||
results = tmp_path / "results"
|
||||
(results / "r2" / "c-facts").mkdir(parents=True)
|
||||
assert seed(manifest, stored, results_root=results) == []
|
||||
(results / "r2" / "c-facts").rmdir()
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
(stored.root / "facts" / f"{sha}.json").write_bytes(b"changed underneath")
|
||||
assert seed(manifest, stored, results_root=results) == []
|
||||
assert not (stored.root / "facts" / f"{sha}.json").exists()
|
||||
@@ -756,6 +756,22 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None:
|
||||
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
|
||||
|
||||
|
||||
def test_mark_done_sets_the_state_under_the_run_lock(tmp_path: Path) -> None:
|
||||
import os
|
||||
|
||||
base = tmp_path / "r"
|
||||
base.mkdir()
|
||||
(base / "lock").write_text(json.dumps(os.getppid())) # a live process that is not this one holds the run
|
||||
with pytest.raises(RuntimeError, match="run is held by pid"):
|
||||
reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path)
|
||||
assert not (base / "state.json").exists()
|
||||
(base / "lock").unlink()
|
||||
reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path)
|
||||
task = json.loads((base / "state.json").read_text())["tasks"]["c-facts"]
|
||||
assert (task["status"], task["reused"]) == ("done", {"sha256": "ab", "from_run": "r0"}) and task["finished"]
|
||||
assert not (base / "lock").exists(), "the lock is released"
|
||||
|
||||
|
||||
def test_reconciler_is_under_500_lines_and_holds_no_pipeline_logic() -> None:
|
||||
text = RECONCILER.read_text(encoding="utf-8")
|
||||
assert len(text.splitlines()) < 500, "Req 3.4: the reconciler stays under 500 lines"
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
"""`remember` (harnessx-removal §4) on a recorded Run: a few KB of validation-46's lmdeploy case under
|
||||
``tests/fixtures/plane-remember`` (witness texts cut, facts.json cut to the files its callers name, the repository
|
||||
URL replaced: the population is reserved, ``tests/test_independent_population.py``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.plane import memory
|
||||
from openultrasast.plane.memory import FileStore, MemoryStoreError, ingest, row_id
|
||||
from openultrasast.plane.tasks import remember
|
||||
from openultrasast.plane.tasks.remember import Context, candidates_digest, remember_run, rows_for
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||
CASE = "lmdeploy-media-url-ssrf"
|
||||
PIN = "71648d239c978452720a6d6692fe914efc8a7b94"
|
||||
IMAGE = "sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "plane" / "validation-46"
|
||||
shutil.copytree(FIXTURE, target)
|
||||
return target
|
||||
|
||||
|
||||
def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
|
||||
rows: dict[str, list[dict[str, Any]]] = {}
|
||||
for record in store.rows():
|
||||
rows.setdefault(record.row["kind"], []).append(record.row)
|
||||
return rows
|
||||
|
||||
|
||||
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
|
||||
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
assert len(facts) > 100, "the fixture was read"
|
||||
store = FileStore(tmp_path / "memory")
|
||||
results = remember_run(run_dir, store, population="population-v2", split="validation")
|
||||
assert [(r.task, r.skipped, r.kinds) for r in results] == [(f"{CASE}-remember", False, {"facts": 1, "unit_cost": 2, "verdict": 1})]
|
||||
rows = _by_kind(store)
|
||||
for row in (r for kind in rows.values() for r in kind):
|
||||
assert (row["repo"], row["pin"], row["run"], row["image"]) == ("example.com/reserved/lmdeploy", PIN, "validation-46", IMAGE)
|
||||
assert (row["task"], row["population"], row["split"]) == (f"{CASE}-remember", "population-v2", "validation")
|
||||
sha = hashlib.sha256(facts).hexdigest()
|
||||
assert rows["facts"][0]["sha256"] == sha and store.get_facts(sha) == facts
|
||||
assert rows["facts"][0]["candidates_digest"] == candidates_digest(json.loads(facts))
|
||||
agreed = json.loads((run_dir / f"{CASE}-final" / "agreed.json").read_text())["candidates"]
|
||||
assert [(v["candidate"], v["passes"], v["site_match"], v["usd"], v["turns"]) for v in rows["verdict"]] == [
|
||||
(c["candidate"], {"a": c["a"], "b": c["b"], "c": c["c"]}, c["site_match"], c["usd"], c["turns"]) for c in agreed
|
||||
], "a verdict row per agree row, from the final (tie-break) decision"
|
||||
assert rows["verdict"][0]["final"] == "agreed" and rows["verdict"][0]["family"] == "untrusted_destination"
|
||||
costs = {(u["pass"], u["path"]): (u["usd"], u["calls"], u["model"]) for u in rows["unit_cost"]}
|
||||
assert costs == {
|
||||
("a", "lmdeploy/vl/media/connection.py"): (0.01237569, 7, "deepseek-flash"),
|
||||
("b", "lmdeploy/vl/media/connection.py"): (costs[("b", "lmdeploy/vl/media/connection.py")][0], 7, "deepseek-flash"),
|
||||
}
|
||||
assert sum(u["usd"] for u in rows["unit_cost"]) == pytest.approx(rows["verdict"][0]["usd"], abs=1e-6)
|
||||
assert not list(run_dir.glob("*-remember")), "the by-hand path writes nothing into the run directory"
|
||||
|
||||
|
||||
def test_re_ingest_of_the_same_run_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
remember_run(run_dir, store)
|
||||
before = {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()}
|
||||
again = remember_run(run_dir, store)
|
||||
assert [r.skipped for r in again] == [True]
|
||||
assert {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()} == before
|
||||
|
||||
|
||||
def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
agreed = {
|
||||
"family": "injection",
|
||||
"candidates": [
|
||||
{"candidate": "a.py::f", "a": True, "b": True, "c": None, "agreed": True},
|
||||
{"candidate": "a.py::g", "a": True, "b": False, "c": True, "agreed": True},
|
||||
{"candidate": "a.py::h", "a": True, "b": False, "c": False, "agreed": False},
|
||||
{"candidate": "a.py::i", "a": False, "b": False, "c": None, "agreed": False},
|
||||
],
|
||||
"disputed": [{"candidate": "a.py::h"}],
|
||||
}
|
||||
alert = {"rule_id": "py-ssrf", "rule_status": "shadow", "path": "a.py", "line": 3, "function": "f", "pin_role": "vulnerable"}
|
||||
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert])
|
||||
verdicts = {r["candidate"]: (r["final"], r["tiebreak"]) for r in rows if r["kind"] == "verdict"}
|
||||
expected = {"a.py::f": ("agreed", False), "a.py::g": ("agreed", True), "a.py::h": ("disputed", True), "a.py::i": ("rejected", False)}
|
||||
assert verdicts == expected
|
||||
(alert_row,) = [r for r in rows if r["kind"] == "alert"]
|
||||
assert {k: alert_row[k] for k in alert} == alert and alert_row["repo"] == "github.com/o/r"
|
||||
assert alert_row["id"] == row_id("alert", "r", "c-remember", "py-ssrf:a.py:3")
|
||||
assert len({r["id"] for r in rows}) == len(rows)
|
||||
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
|
||||
|
||||
|
||||
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
|
||||
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
|
||||
task = next(d for d in docs if d["kind"] == "Task")
|
||||
pins = next(e["value"] for e in task["spec"]["env"] if e["name"] == "OUSAST_GIT_PINS")
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_RUN": "validation-46", "OUSAST_TASK": f"{CASE}-remember", "OUSAST_GIT_PINS": pins,
|
||||
"AX_TASK_YAML": yaml.safe_dump(task), "AX_WORKSPACES_YAML": yaml.safe_dump_all([d for d in docs if d["kind"] == "Workspace"]),
|
||||
"OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"),
|
||||
"OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"),
|
||||
"OUSAST_POPULATION": "population-v2", "OUSAST_SPLIT": "validation",
|
||||
**{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"},
|
||||
**{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"},
|
||||
} # fmt: skip
|
||||
assert remember.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["units_done"], summary["usd"], summary["calls"], summary["rows"]) == ("done", 1, 0, 0, 4)
|
||||
assert (out / "facts.json").read_bytes() == (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
delivered = FileStore(tmp_path / "delivered")
|
||||
assert [r.task for r in ingest(run_dir, delivered)] == [f"{CASE}-remember"]
|
||||
by_hand = FileStore(tmp_path / "by-hand")
|
||||
(out / "memory.jsonl").rename(tmp_path / "memory.jsonl")
|
||||
remember_run(run_dir, by_hand, population="population-v2", split="validation")
|
||||
assert [r.row for r in delivered.rows()] == [r.row for r in by_hand.rows()], "the task and the host path agree"
|
||||
assert remember.main({**env, "AX_TASK_YAML": "{}"}) == 2
|
||||
assert "names no image" in json.loads((out / "summary.json").read_text())["reason"]
|
||||
|
||||
|
||||
def test_ingest_without_remember_outputs_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
assert ingest(run_dir, store) == [] and not store.root.exists()
|
||||
|
||||
|
||||
def test_a_malformed_delivered_row_fails_naming_file_and_line(run_dir: Path, tmp_path: Path) -> None:
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
out.mkdir()
|
||||
good = rows_for(Context("validation-46", out.name, "https://github.com/o/r", PIN, IMAGE), facts=None, passes={}, models={}, agreed=None,
|
||||
alerts=[{"rule_id": "x", "path": "a.py", "line": 1}]) # fmt: skip
|
||||
(out / "memory.jsonl").write_text(json.dumps(good[0]) + "\n" + json.dumps({**good[0], "kind": "vibes"}) + "\n")
|
||||
store = FileStore(tmp_path / "memory")
|
||||
with pytest.raises(MemoryStoreError, match=rf"{out.name}/memory.jsonl:2: unknown kind"):
|
||||
remember_run(run_dir, store)
|
||||
assert not store.root.exists()
|
||||
|
||||
|
||||
def test_plane_remember_cli_reads_population_from_the_run_and_reports_counts(
|
||||
run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "1 tasks ingested, 0 already in the index; rows facts=1 unit_cost=2 verdict=1" in out
|
||||
rows = FileStore(tmp_path / "memory").rows()
|
||||
assert {(r.row["population"], r.row["split"]) for r in rows} == {("population-v2", "validation")} # plane/runs/validation-46.yaml
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
assert "0 tasks ingested, 1 already in the index" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_plane_run_seeds_before_and_ingests_after(run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.plane import reconciler
|
||||
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(memory, "seed", lambda manifest, store: calls.append(f"seed {manifest.name}") or [])
|
||||
monkeypatch.setattr(reconciler, "run", lambda manifest, **kw: calls.append("run") or "done")
|
||||
assert cli.main(["plane", "run", "plane/runs/validation-46.yaml"]) == 0
|
||||
assert calls == ["seed validation-46.yaml", "run"]
|
||||
assert len(FileStore(tmp_path / "memory").rows(kind="verdict")) == 1, "the run's rows were ingested after it"
|
||||
@@ -216,6 +216,65 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "argon2-cffi"
|
||||
version = "25.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "argon2-cffi-bindings" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "argon2-cffi-bindings"
|
||||
version = "26.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cffi" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/5f/f652055e18d2627e2eed94c7f31a792127cfe38df786635395d742321674/argon2_cffi_bindings-26.1.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083", size = 15434, upload-time = "2026-08-20T07:32:53.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/cb/214092c39c4dbcb72cf98b12234ddac2221f8fe2c0acf29c6a70fa83be53/argon2_cffi_bindings-26.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb", size = 25683, upload-time = "2026-08-20T07:33:05.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/e5/02015b83e9b05ccb85ff2ced424cf6e83a12d3810bc7f66d679a92b69ffb/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6", size = 27310, upload-time = "2026-08-20T07:33:06.344Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/4a/85e612787d0796878b3b4f6bd53dcd5484b6fe7b64cc6fc7b6e6a04cf835/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990", size = 26771, upload-time = "2026-08-20T07:33:07.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/84/ccb003b6f9969820e87656398f4d49c857def71a85ca1588a0e809afd7ce/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08", size = 27569, upload-time = "2026-08-20T07:33:08.598Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/07/c26b76debf0998ee08fbe947ab2058ac5de37d4b9d46b06c17abaa6c4ce9/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca", size = 27279, upload-time = "2026-08-20T07:33:09.518Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/0d/ead6ddc029f91bc9b9390686dad3c808ab08100d348f6266b5f93f8970ee/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1", size = 27774, upload-time = "2026-08-20T07:33:10.728Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/47/c108530d9eb86036b78d3af4de28b83b4a2d9a70512bd10ff8e59966aab4/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36", size = 26933, upload-time = "2026-08-20T07:33:11.661Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/02/0bfc59e781c89acf64c31c388aade9d9d1c1ea38aa1ba1292fe07f607fe9/argon2_cffi_bindings-26.1.0-cp315-cp315t-win32.whl", hash = "sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210", size = 24875, upload-time = "2026-08-20T07:33:12.616Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/c7/c3e46068cddffccecb8ad94d71135e9bf62bbc789589e7dfadc7c6f59214/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4", size = 26655, upload-time = "2026-08-20T07:33:13.521Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/ca/18b9c8c45fecf34b9100ec6d7946057f14a158f2eaa20ea123a3e82351cb/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440", size = 25376, upload-time = "2026-08-20T07:33:14.491Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/b9/97f0370f99611b14efd384918613dd5cbda75f28d9bb1b677aacfeaa17df/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8", size = 23055, upload-time = "2026-08-20T07:33:19.716Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/70/7eb3fe7bf00103cbbb569c51aef150661f22b734a782673a600ff0f52309/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a", size = 24869, upload-time = "2026-08-20T07:33:20.671Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/4b/9d5919c6cb1f15df7406af0f99b048bd93936f112e3e8f4c8077bc2a9110/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba", size = 24216, upload-time = "2026-08-20T07:33:21.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ast-serialize"
|
||||
version = "0.5.0"
|
||||
@@ -1765,6 +1824,22 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minio"
|
||||
version = "7.2.20"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "argon2-cffi" },
|
||||
{ name = "certifi" },
|
||||
{ name = "pycryptodome" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multidict"
|
||||
version = "6.7.1"
|
||||
@@ -2140,11 +2215,17 @@ wheels = [
|
||||
name = "openultrasast"
|
||||
version = "1.2.0a1"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "pyyaml" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
harnessx = [
|
||||
{ name = "harnessx" },
|
||||
]
|
||||
minio = [
|
||||
{ name = "minio" },
|
||||
]
|
||||
semantic = [
|
||||
{ name = "tree-sitter" },
|
||||
{ name = "tree-sitter-c" },
|
||||
@@ -2167,6 +2248,8 @@ dev = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "harnessx", marker = "extra == 'harnessx'", git = "https://github.com/Darwin-Agent/HarnessX.git?rev=274c512eea67c64701f992d3b6fe06f0fd590ef0" },
|
||||
{ name = "minio", marker = "extra == 'minio'", specifier = ">=7.2,<8" },
|
||||
{ name = "pyyaml", specifier = ">=6.0" },
|
||||
{ name = "tree-sitter", marker = "extra == 'semantic'", specifier = ">=0.23,<0.27" },
|
||||
{ name = "tree-sitter-c", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
{ name = "tree-sitter-cpp", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
@@ -2175,7 +2258,7 @@ requires-dist = [
|
||||
{ name = "tree-sitter-python", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
{ name = "tree-sitter-typescript", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
]
|
||||
provides-extras = ["harnessx", "semantic"]
|
||||
provides-extras = ["harnessx", "minio", "semantic"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
@@ -2518,6 +2601,36 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycryptodome"
|
||||
version = "3.23.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "2.13.4"
|
||||
|
||||
Reference in New Issue
Block a user