mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): memory store (file and MinIO) and the remember task (harnessx-removal task 2)
plane/memory.py: the MemoryStore interface with FileStore (index.jsonl, facts by content hash, repos/<host>__<owner>__<name>/<pin>.jsonl, idempotent ingest, deterministic row ids, refusal below 1 GiB free naming the path) and MinioStore (object metadata and tags, S3 Select pushdown probed once per process with a logged local fallback, object version ids on every row read, versioning and runs/ lifecycle helper, presigned URLs). OUSAST_MEMORY selects the backend; MINIO_* settings come from .env; the minio SDK is the optional extra `minio`. plane/tasks/remember.py: model-free rows (facts, verdict, unit_cost, alert) from a case's artifacts; remember_run computes them on the host for a recorded run without writing into it. reconciler.mark_done (477 lines total), seed before and ingest after `ousast plane run`, `ousast plane remember <run>`. The generator annotates each repo-facts Task with openultrasast.io/memory-key (repo, pin, candidates digest, image digest) and the Run with its population and split; plane/ regenerated. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
d64ab2817e
commit
52d28b5121
+59
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"family": "untrusted_destination",
|
||||
"candidates": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"a": true,
|
||||
"b": true,
|
||||
"agreed": true,
|
||||
"disputed": false,
|
||||
"site_match": true,
|
||||
"usd": 0.022175,
|
||||
"turns": 10,
|
||||
"finding": {
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"agreed": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c",
|
||||
"passes": 2
|
||||
}
|
||||
],
|
||||
"disputed": [],
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[]
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": null,
|
||||
"pass": null,
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"callers": {
|
||||
"lmdeploy/vl/media/connection.py::_load_http_url": []
|
||||
},
|
||||
"counts": {
|
||||
"callers": 0,
|
||||
"candidates": 1,
|
||||
"files": 1,
|
||||
"functions": 4809
|
||||
},
|
||||
"files": {
|
||||
"lmdeploy/vl/media/connection.py": {
|
||||
"functions": [
|
||||
"_is_safe_url",
|
||||
"_load_http_url",
|
||||
"_load_data_url",
|
||||
"_load_file_url",
|
||||
"load_from_url"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"calls": 0,
|
||||
"model": null,
|
||||
"status": "done",
|
||||
"units_done": 608,
|
||||
"units_total": 608,
|
||||
"usage": {},
|
||||
"usd": null
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Workspace
|
||||
metadata:
|
||||
name: lmdeploy-media-url-ssrf-vulnerable
|
||||
atespace: default
|
||||
spec:
|
||||
git:
|
||||
- name: repo
|
||||
repo: https://example.com/reserved/lmdeploy
|
||||
dir: repo
|
||||
depth: 1
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: validation-46-lmdeploy-media-url-ssrf-facts
|
||||
atespace: default
|
||||
spec:
|
||||
command:
|
||||
- repo-facts
|
||||
env:
|
||||
- name: OUSAST_RUN
|
||||
value: validation-46
|
||||
- name: OUSAST_TASK
|
||||
value: lmdeploy-media-url-ssrf-facts
|
||||
- name: OUSAST_GIT_PINS
|
||||
value: '{"lmdeploy-media-url-ssrf-vulnerable/repo": "71648d239c978452720a6d6692fe914efc8a7b94"}'
|
||||
image: localhost:5001/ousast-runner@sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1280Mi
|
||||
workspaces:
|
||||
- name: lmdeploy-media-url-ssrf-vulnerable
|
||||
path: /workspace/case
|
||||
- name: lmdeploy-media-url-ssrf-inputs
|
||||
path: /workspace/inputs
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"family": "untrusted_destination",
|
||||
"candidates": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"a": true,
|
||||
"b": true,
|
||||
"c": null,
|
||||
"agreed": true,
|
||||
"disputed": false,
|
||||
"site_match": true,
|
||||
"usd": 0.022175,
|
||||
"turns": 10,
|
||||
"finding": {
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"agreed": [
|
||||
{
|
||||
"candidate": "lmdeploy/vl/media/connection.py::_load_http_url",
|
||||
"site": "lmdeploy/vl/media/connection.py:68:_load_http_url",
|
||||
"reported_at": "lmdeploy/vl/media/connection.py:68",
|
||||
"family": "untrusted_destination",
|
||||
"title": "SSRF guard bypass via HTTP redirect in _load_http_url",
|
||||
"witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c",
|
||||
"passes": 2
|
||||
}
|
||||
],
|
||||
"disputed": [],
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0,
|
||||
"tiebreak": {
|
||||
"asked": 0,
|
||||
"flipped_to_agreed": 0,
|
||||
"flipped_to_rejected": 0,
|
||||
"rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[]
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": null,
|
||||
"pass": null,
|
||||
"metrics": {
|
||||
"candidates": 1,
|
||||
"candidates_before_triage": 1,
|
||||
"asked_in_both": 1,
|
||||
"flagged_a": 1,
|
||||
"flagged_b": 1,
|
||||
"agreed": 1,
|
||||
"disputed": 0,
|
||||
"hunts": 2,
|
||||
"declared_sites": 1,
|
||||
"declared_sites_in_set": 1,
|
||||
"declared_sites_matched": 1,
|
||||
"declared_sites_matched_agreed": 1,
|
||||
"site_matching": "assessed",
|
||||
"usd_total": 0.022175,
|
||||
"cost_per_candidate": 0.022175,
|
||||
"recorded_triage_usd": 0.000768,
|
||||
"recorded_triage_basis": "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage priced at deepseek-flash list prices; the triage call's own usage was not recorded",
|
||||
"usd_total_with_recorded_triage": 0.022943,
|
||||
"cost_per_candidate_with_recorded_triage": 0.022943,
|
||||
"turns_per_hunt": 5.0,
|
||||
"calls_per_hunt": 7.0,
|
||||
"tiebreak": {
|
||||
"asked": 0,
|
||||
"flipped_to_agreed": 0,
|
||||
"flipped_to_rejected": 0,
|
||||
"rule": "a candidate asked in pass c is agreed when at least 2 of passes a, b, c flagged it; a candidate not asked in pass c is agreed when passes a and b both flagged it (pass c alone never decides)"
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0.012376,
|
||||
"calls": 7,
|
||||
"usage": {
|
||||
"prompt_tokens": 68714,
|
||||
"prompt_cache_hit_tokens": 48512,
|
||||
"completion_tokens": 2127
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "a",
|
||||
"family": "untrusted_destination"
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "a", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF guard bypass via HTTP redirect in _load_http_url", "witness": "Source: an attacker-controlled HTTP request to the OpenAI-compatible RESTful API whose `messages` contain a multimodal c"}], "turns": 5, "usage": {"prompt_tokens": 68714, "prompt_cache_hit_tokens": 48512, "completion_tokens": 2127, "calls": 7}, "usd": 0.01237569}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 1,
|
||||
"units_total": 1,
|
||||
"usd": 0.009799,
|
||||
"calls": 7,
|
||||
"usage": {
|
||||
"prompt_tokens": 61163,
|
||||
"prompt_cache_hit_tokens": 46464,
|
||||
"completion_tokens": 2031
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "b",
|
||||
"family": "untrusted_destination"
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"path": "lmdeploy/vl/media/connection.py", "candidates": [["lmdeploy/vl/media/connection.py", "_load_http_url", 68]], "pass": "b", "flagged": [{"candidate": "lmdeploy/vl/media/connection.py::_load_http_url", "site": "lmdeploy/vl/media/connection.py:68:_load_http_url", "reported_at": "lmdeploy/vl/media/connection.py:68", "family": "untrusted_destination", "title": "SSRF via unvalidated HTTP redirects in media URL fetch", "witness": "Source: an HTTP client submits a chat request whose multimodal content contains an attacker-controlled URL, e.g. {'type'"}], "turns": 5, "usage": {"prompt_tokens": 61163, "prompt_cache_hit_tokens": 46464, "completion_tokens": 2031, "calls": 7}, "usd": 0.00979898}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"status": "done",
|
||||
"units_done": 0,
|
||||
"units_total": 0,
|
||||
"usd": 0.0,
|
||||
"calls": 0,
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"completion_tokens": 0
|
||||
},
|
||||
"priced": true,
|
||||
"model": "deepseek-flash",
|
||||
"pass": "c",
|
||||
"family": "untrusted_destination",
|
||||
"only": 0
|
||||
}
|
||||
@@ -19,6 +19,8 @@ from openultrasast.plane import reconciler
|
||||
from openultrasast.plane.budget import prices_from
|
||||
from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates
|
||||
from openultrasast.plane.manifests import load_manifests
|
||||
from openultrasast.plane.memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, parse_memory_key
|
||||
from openultrasast.plane.tasks.repo_facts import _digest
|
||||
|
||||
PLANE = Path("plane")
|
||||
RUN = PLANE / "runs" / "validation-46.yaml"
|
||||
@@ -160,7 +162,8 @@ def test_committed_manifests_match_the_generator() -> None:
|
||||
POPULATION,
|
||||
REPOS,
|
||||
)
|
||||
for relative, text in render(cases, templates, "validation-46", _committed_command()).items():
|
||||
rendered = render(cases, templates, "validation-46", _committed_command(), population=POPULATION.stem, split="validation")
|
||||
for relative, text in rendered.items():
|
||||
assert (PLANE / relative).read_text() == text, relative
|
||||
|
||||
|
||||
@@ -216,6 +219,12 @@ def test_generator_is_deterministic_and_follows_the_reference(tmp_path: Path) ->
|
||||
assert inputs["triage.json"]["candidates"] == [["a.py", "helper", 20], ["a.py", "run", 9]]
|
||||
assert run.task("alpha-va").budget is not None and run.task("alpha-va").budget.usd == 0.38 # 1.5 * 0.5 / 2, cent up
|
||||
assert yaml.safe_load(first[0].read_text())["metadata"]["annotations"] == {"openultrasast.io/git-commits": f"repo={PIN}"}
|
||||
# memory (harnessx-removal §4): the facts Task names its reuse key, the Run its population and split
|
||||
key = parse_memory_key(manifests.tasks["repo-facts-alpha"].metadata.annotations[MEMORY_KEY_ANNOTATION])
|
||||
image = manifests.tasks["repo-facts-alpha"].image or ""
|
||||
assert key == {"repo": "example.com/alpha", "pin": PIN, "candidates": _digest(["run"]), "image": image.rpartition("@")[2]}
|
||||
assert all(MEMORY_KEY_ANNOTATION not in t.metadata.annotations for n, t in manifests.tasks.items() if not n.startswith("repo-facts-"))
|
||||
assert dict(run.metadata.annotations) == {POPULATION_ANNOTATION: "population", SPLIT_ANNOTATION: "set"}
|
||||
|
||||
|
||||
def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path) -> None:
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
"""The plane's memory store (harnessx-removal Req 6.1/6.4, design §4): one contract over every backend, the S3
|
||||
Select probe and its fallback on a fake object client, backend selection, fact reuse by ``seed``, disk refusal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.plane import memory, reconciler
|
||||
from openultrasast.plane.memory import (
|
||||
MINIO_KEYS,
|
||||
FileStore,
|
||||
MemoryStore,
|
||||
MemoryStoreError,
|
||||
MinioStore,
|
||||
SdkClient,
|
||||
memory_key,
|
||||
minio_settings,
|
||||
open_store,
|
||||
row_id,
|
||||
seed,
|
||||
where_sql,
|
||||
)
|
||||
from openultrasast.plane.tasks.repo_facts import _digest
|
||||
|
||||
PIN = "0123456789abcdef0123456789abcdef01234567"
|
||||
OTHER_PIN = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
IMAGE = "sha256:" + "a" * 64
|
||||
FACTS = json.dumps({"callers": {"a.py::run": []}, "counts": {"files": 1}, "files": {"a.py": {"functions": ["run"]}}}).encode()
|
||||
MINIO_SKIP = (
|
||||
"MinIO contract tests need OUSAST_MEMORY_TEST_MINIO=1 plus MINIO_ENDPOINT, MINIO_ACCESS_KEY and MINIO_SECRET_KEY "
|
||||
"exported (optional: MINIO_SECURE, OUSAST_MEMORY_TEST_BUCKET, default ousast-memory-test) and the minio extra installed"
|
||||
)
|
||||
|
||||
|
||||
def row(kind: str, subject: str, *, pin: str = PIN, run: str = "r1", repo: str = "https://github.com/o/r", **fields: Any) -> dict[str, Any]:
|
||||
task = "c-remember"
|
||||
base = {"id": row_id(kind, run, task, subject), "kind": kind, "repo": memory.repo_key(repo), "pin": pin, "run": run, "task": task}
|
||||
return {**base, "population": "population-v2", "split": "validation", "image": IMAGE, **fields}
|
||||
|
||||
|
||||
def verdict(candidate: str, final: str, **fields: Any) -> dict[str, Any]:
|
||||
return row("verdict", candidate, candidate=candidate, family="injection", final=final, tiebreak=False, **fields)
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
"""An in-memory :class:`memory.ObjectClient`: versions per write, tags per object; ``select`` works, raises or
|
||||
answers wrongly, evaluating the equality clauses :func:`where_sql` writes."""
|
||||
|
||||
def __init__(self, select: str = "works") -> None:
|
||||
self.mode, self.objects, self.versions = select, {}, {}
|
||||
self.selects: list[str] = []
|
||||
self.gets: list[str] = []
|
||||
self.calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def get(self, key: str) -> tuple[bytes, str | None] | None:
|
||||
self.gets.append(key)
|
||||
found = self.objects.get(key)
|
||||
return None if found is None else (found[0], f"v{self.versions[key]}")
|
||||
|
||||
def put(self, key: str, data: bytes, labels: dict[str, str]) -> None:
|
||||
self.versions[key] = self.versions.get(key, 0) + 1
|
||||
self.objects[key] = (data, dict(labels))
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.objects.pop(key, None)
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
return sorted(k for k in self.objects if k.startswith(prefix))
|
||||
|
||||
def tags(self, key: str) -> dict[str, str]:
|
||||
return dict(self.objects[key][1])
|
||||
|
||||
def version(self, key: str) -> str | None:
|
||||
return f"v{self.versions[key]}" if key in self.objects else None
|
||||
|
||||
def select(self, key: str, expression: str) -> bytes:
|
||||
self.selects.append(expression)
|
||||
if self.mode == "raises":
|
||||
raise RuntimeError("NotImplemented: SelectObjectContent is not supported")
|
||||
if self.mode == "wrong":
|
||||
return b""
|
||||
clauses = re.findall(r"s\.\"(\w+)\" = ('(?:[^']|'')*'|true|false|-?[\d.]+)", expression)
|
||||
want = {n: v[1:-1].replace("''", "'") if v.startswith("'") else json.loads(v) for n, v in clauses}
|
||||
rows = [json.loads(line) for line in self.objects[key][0].splitlines() if line.strip()]
|
||||
return b"".join(json.dumps(r).encode() + b"\n" for r in rows if all(r.get(k) == v for k, v in want.items()))
|
||||
|
||||
def presign(self, method: str, key: str, expires: timedelta) -> str:
|
||||
return f"https://minio.test/{key}?method={method}&X-Amz-Expires={int(expires.total_seconds())}"
|
||||
|
||||
def expire(self, prefix: str, days: int, rule_id: str) -> None:
|
||||
self.calls.append(("expire", prefix, days, rule_id))
|
||||
|
||||
def enable_versioning(self) -> None:
|
||||
self.calls.append(("versioning",))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh_probes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(memory, "_SELECT_PROBES", {})
|
||||
|
||||
|
||||
def _real_minio() -> Iterator[MemoryStore]:
|
||||
if os.environ.get("OUSAST_MEMORY_TEST_MINIO") != "1" or any(not os.environ.get(k) for k in MINIO_KEYS):
|
||||
pytest.skip(MINIO_SKIP)
|
||||
pytest.importorskip("minio", reason=MINIO_SKIP)
|
||||
bucket = os.environ.get("OUSAST_MEMORY_TEST_BUCKET") or "ousast-memory-test"
|
||||
client = SdkClient(bucket=bucket, **minio_settings(os.environ))
|
||||
if not client.sdk.bucket_exists(bucket):
|
||||
client.sdk.make_bucket(bucket)
|
||||
store = MinioStore(client, bucket, f"contract-{uuid.uuid4().hex[:12]}")
|
||||
client.enable_versioning()
|
||||
yield store
|
||||
for key in client.keys(store.prefix + "/"):
|
||||
client.delete(key)
|
||||
|
||||
|
||||
@pytest.fixture(params=["file", "fake-minio-select", "fake-minio-fallback", "minio"])
|
||||
def store(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[MemoryStore]:
|
||||
if request.param == "file":
|
||||
yield FileStore(tmp_path / "memory")
|
||||
elif request.param == "minio":
|
||||
yield from _real_minio()
|
||||
else:
|
||||
yield MinioStore(FakeObjects("works" if request.param.endswith("select") else "raises"), "bucket", "p")
|
||||
|
||||
|
||||
# --- the contract --------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ingest_is_idempotent_by_the_index(store: MemoryStore) -> None:
|
||||
rows = [verdict("a.py::run", "agreed"), verdict("a.py::other", "rejected")]
|
||||
first = store.ingest_rows("r1", "c-remember", rows, FACTS)
|
||||
again = store.ingest_rows("r1", "c-remember", rows, FACTS)
|
||||
assert (first.skipped, again.skipped, first.kinds) == (False, True, {"verdict": 2})
|
||||
assert [(e["run"], e["task"], e["rows"]) for e in store.index()] == [("r1", "c-remember", 2)]
|
||||
assert len(store.rows(kind="verdict")) == 2
|
||||
|
||||
|
||||
def test_identical_facts_are_stored_once_and_read_back_by_content(store: MemoryStore) -> None:
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
store.ingest_rows("r1", "c-remember", [row("facts", sha, sha256=sha)], FACTS)
|
||||
store.ingest_rows("r2", "c-remember", [row("facts", sha, run="r2", sha256=sha)], FACTS)
|
||||
assert store._keys("facts/") == [f"facts/{sha}.json"]
|
||||
assert store.get_facts(sha) == FACTS and store.get_facts("0" * 64) is None
|
||||
|
||||
|
||||
def test_rows_filter_by_repo_pin_kind_and_where_with_their_object_version(store: MemoryStore) -> None:
|
||||
store.put_rows([
|
||||
verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), verdict("b.py::y", "agreed", pin=OTHER_PIN),
|
||||
verdict("c.py::z", "agreed", repo="https://github.com/o/other.git"), row("unit_cost", "a:a.py", **{"pass": "a", "usd": 0.5}),
|
||||
]) # fmt: skip
|
||||
assert len(store.rows()) == 5
|
||||
assert {r.row["candidate"] for r in store.rows("https://github.com/o/r", kind="verdict")} == {"a.py::run", "a.py::x", "b.py::y"}
|
||||
assert [r.row["candidate"] for r in store.rows("github.com/o/r", PIN, "verdict", {"final": "disputed"})] == ["a.py::x"]
|
||||
assert [r.row["kind"] for r in store.rows(where={"usd": 0.5})] == ["unit_cost"]
|
||||
assert store.rows(where={"final": "no such"}) == []
|
||||
record = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0]
|
||||
assert record.key == f"repos/github.com__o__r/{PIN}.jsonl" and record.version
|
||||
store.put_row(verdict("a.py::new", "agreed"))
|
||||
later = store.rows("github.com/o/r", PIN, "verdict", {"candidate": "a.py::run"})[0]
|
||||
assert later.row == record.row and later.version != record.version, "a rewrite is a new version: provenance pins the old"
|
||||
|
||||
|
||||
def test_a_repeated_id_replaces_the_stored_row(store: MemoryStore) -> None:
|
||||
store.put_row(verdict("a.py::run", "disputed"))
|
||||
store.put_row(verdict("a.py::run", "agreed"))
|
||||
assert [r.row["final"] for r in store.rows(kind="verdict")] == ["agreed"]
|
||||
|
||||
|
||||
def test_a_malformed_row_fails_naming_it_and_writes_nothing(store: MemoryStore) -> None:
|
||||
bad = {**verdict("a.py::run", "agreed"), "pin": "main"}
|
||||
with pytest.raises(MemoryStoreError, match=r"r1/c-remember row 2: pin 'main'"):
|
||||
store.ingest_rows("r1", "c-remember", [verdict("a.py::ok", "agreed"), bad])
|
||||
with pytest.raises(MemoryStoreError, match="unknown kind"):
|
||||
store.put_row({**verdict("a.py::run", "agreed"), "kind": "vibes"})
|
||||
assert store.index() == [] and store.rows() == []
|
||||
|
||||
|
||||
def test_facts_that_no_longer_match_their_hash_are_dropped(store: MemoryStore, caplog: pytest.LogCaptureFixture) -> None:
|
||||
sha = store.put_facts(FACTS)
|
||||
store._put(f"facts/{sha}.json", b"tampered")
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
assert store.get_facts(sha) is None
|
||||
assert "no longer matches" in caplog.text and store._get(f"facts/{sha}.json") is None
|
||||
|
||||
|
||||
def test_presigned_urls_name_the_object(store: MemoryStore) -> None:
|
||||
for url in (store.presign_put("runs/r1/x/units.jsonl"), store.presign_get("runs/r1/x/units.jsonl")):
|
||||
assert url.startswith(("file://", "http://", "https://")) and "runs/r1/x/units.jsonl" in url
|
||||
|
||||
|
||||
# --- MinIO specifics on the fake client ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _filled(client: FakeObjects) -> MinioStore:
|
||||
store = MinioStore(client, "bucket")
|
||||
store.put_rows([verdict("a.py::run", "agreed"), verdict("a.py::x", "disputed"), row("unit_cost", "a:a.py", **{"pass": "a"})])
|
||||
return store
|
||||
|
||||
|
||||
def test_select_is_pushed_down_when_the_probe_answers(caplog: pytest.LogCaptureFixture) -> None:
|
||||
client = FakeObjects("works")
|
||||
store = _filled(client)
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
got = [r.row["candidate"] for r in store.rows(kind="verdict", where={"final": "disputed"})]
|
||||
assert got == ["a.py::x"] and store.select_supported() and "locally" not in caplog.text
|
||||
assert client.selects[0] == 'SELECT * FROM S3Object s WHERE s."probe" = 1'
|
||||
assert client.selects[1:] == ["SELECT * FROM S3Object s WHERE s.\"final\" = 'disputed' AND s.\"kind\" = 'verdict'"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["raises", "wrong"])
|
||||
def test_without_select_the_same_rows_come_from_a_local_filter_logged_once(
|
||||
mode: str, caplog: pytest.LogCaptureFixture, tmp_path: Path
|
||||
) -> None:
|
||||
client = FakeObjects(mode)
|
||||
store, reference = _filled(client), FileStore(tmp_path / "m")
|
||||
reference.put_rows([r.row for r in store.rows()])
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
for where in ({"final": "disputed"}, {"final": "agreed"}, {"pass": "a"}):
|
||||
assert [r.row for r in store.rows(where=where)] == [r.row for r in reference.rows(where=where)] != []
|
||||
assert not store.select_supported() and len(client.selects) == 1, "one probe per process"
|
||||
assert caplog.text.count("does not answer S3 Select") == 1
|
||||
|
||||
|
||||
def test_a_failing_select_on_one_object_falls_back_for_that_object(caplog: pytest.LogCaptureFixture) -> None:
|
||||
client = FakeObjects("works")
|
||||
store = _filled(client)
|
||||
assert store.select_supported()
|
||||
client.mode = "raises"
|
||||
with caplog.at_level(logging.WARNING, logger="openultrasast.plane.memory"):
|
||||
assert [r.row["candidate"] for r in store.rows(where={"final": "agreed"})] == ["a.py::run"]
|
||||
assert "filtering it locally" in caplog.text
|
||||
|
||||
|
||||
def test_row_objects_carry_tags_and_a_kind_tag_skips_objects_without_that_kind() -> None:
|
||||
client = FakeObjects("works")
|
||||
store = MinioStore(client, "bucket")
|
||||
store.put_rows([verdict("a.py::run", "agreed"), row("unit_cost", "a:b.py", pin=OTHER_PIN, **{"pass": "a"})])
|
||||
tags = client.tags(f"repos/github.com__o__r/{PIN}.jsonl")
|
||||
assert tags == {"repo": "github.com/o/r", "pin": PIN, "kind": "verdict", "family": "injection", "run": "r1",
|
||||
"population": "population-v2", "split": "validation"} # fmt: skip
|
||||
client.gets.clear()
|
||||
assert [r.row["kind"] for r in store.rows(kind="unit_cost")] == ["unit_cost"]
|
||||
assert f"repos/github.com__o__r/{PIN}.jsonl" not in client.gets, "an object tagged without the kind is not read"
|
||||
|
||||
|
||||
def test_configure_turns_versioning_on_and_expires_runs_only() -> None:
|
||||
client = FakeObjects()
|
||||
MinioStore(client, "bucket", "memory").configure(14)
|
||||
assert client.calls == [("versioning",), ("expire", "memory/runs/", 14, "ousast-runs-expiry")]
|
||||
with pytest.raises(ValueError):
|
||||
MinioStore(client, "bucket").configure(0)
|
||||
|
||||
|
||||
def test_where_sql_quotes_values_and_refuses_odd_fields() -> None:
|
||||
assert where_sql({}) == "SELECT * FROM S3Object s"
|
||||
assert where_sql({"a": "it's", "b": True, "c": None, "d": 2}) == (
|
||||
'SELECT * FROM S3Object s WHERE s."a" = \'it\'\'s\' AND s."b" = true AND s."c" IS NULL AND s."d" = 2'
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
where_sql({"a; drop": 1})
|
||||
|
||||
|
||||
# --- selection and settings ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_open_store_selects_the_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.delenv("OUSAST_MEMORY", raising=False)
|
||||
default = open_store()
|
||||
assert isinstance(default, FileStore) and default.root == tmp_path / "plane" / "memory"
|
||||
chosen = open_store(environ={"OUSAST_MEMORY": f"file://{tmp_path}/m"})
|
||||
assert isinstance(chosen, FileStore) and chosen.root == tmp_path / "m"
|
||||
with pytest.raises(MemoryStoreError, match="file:///<path> or minio://<bucket>"):
|
||||
open_store("s3://bucket")
|
||||
|
||||
|
||||
def test_minio_settings_name_missing_keys_never_values() -> None:
|
||||
with pytest.raises(MemoryStoreError, match="MINIO_ACCESS_KEY, MINIO_SECRET_KEY") as caught:
|
||||
open_store(environ={"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "minio.example:9000"})
|
||||
assert "minio.example" not in str(caught.value)
|
||||
env = {"MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "sk-value", "MINIO_SECURE": "false"}
|
||||
assert minio_settings(env) == {"endpoint": "h:9000", "access_key": "ak", "secret_key": "sk-value", "secure": False}
|
||||
assert minio_settings({**env, "MINIO_SECURE": ""})["secure"] is True
|
||||
|
||||
|
||||
def test_minio_without_the_sdk_names_the_extra() -> None:
|
||||
try:
|
||||
import minio # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
pytest.skip("the minio SDK is installed here")
|
||||
env = {"OUSAST_MEMORY": "minio://bucket", "MINIO_ENDPOINT": "h:9000", "MINIO_ACCESS_KEY": "ak", "MINIO_SECRET_KEY": "secret-xyz"}
|
||||
with pytest.raises(MemoryStoreError, match=r"install openultrasast\[minio\]") as caught:
|
||||
open_store(environ=env)
|
||||
assert "secret-xyz" not in str(caught.value)
|
||||
|
||||
|
||||
# --- disk refusal (FileStore) --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_store_refuses_below_one_gib_free_naming_its_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
root = tmp_path / "memory"
|
||||
monkeypatch.setattr(memory.shutil, "disk_usage", lambda path: shutil._ntuple_diskusage(100 << 30, 99 << 30, 512 << 20)) # type: ignore[attr-defined]
|
||||
with pytest.raises(MemoryStoreError, match=rf"refusing to write the memory store at {re.escape(str(root))}: 0.50 GiB free"):
|
||||
FileStore(root).ingest_rows("r1", "c-remember", [verdict("a.py::run", "agreed")], FACTS)
|
||||
assert not root.exists()
|
||||
|
||||
|
||||
# --- fact reuse ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _plane(root: Path, key: str, run: str = "r2") -> Path:
|
||||
(root / "runs").mkdir(parents=True)
|
||||
(root / "tasks").mkdir()
|
||||
(root / "tasks" / "t.yaml").write_text(
|
||||
"apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: repo-facts-c\n annotations:\n"
|
||||
f" {memory.MEMORY_KEY_ANNOTATION}: '{key}'\nspec:\n image: reg/img@{IMAGE}\n command: [repo-facts]\n"
|
||||
)
|
||||
(root / "runs" / f"{run}.yaml").write_text(
|
||||
f"apiVersion: openultrasast.io/v1alpha1\nkind: Run\nmetadata:\n name: {run}\nspec:\n tasks:\n"
|
||||
" - name: c-facts\n task: repo-facts-c\n outputs: [facts.json]\n"
|
||||
)
|
||||
return root / "runs" / f"{run}.yaml"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stored(tmp_path: Path) -> FileStore:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
facts = row("facts", sha, sha256=sha, candidates_digest=_digest(["run"]), files=1)
|
||||
store.ingest_rows("r1", "c-remember", [facts], FACTS)
|
||||
return store
|
||||
|
||||
|
||||
def test_seed_reuses_facts_for_an_unchanged_key_and_marks_the_task_done(stored: FileStore, tmp_path: Path) -> None:
|
||||
manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}"))
|
||||
results = tmp_path / "results"
|
||||
assert seed(manifest, stored, results_root=results) == ["c-facts"]
|
||||
base = results / "r2"
|
||||
assert (base / "c-facts" / "facts.json").read_bytes() == FACTS
|
||||
summary = json.loads((base / "c-facts" / "summary.json").read_text())
|
||||
reused = {"sha256": hashlib.sha256(FACTS).hexdigest(), "from_run": "r1", "image": IMAGE}
|
||||
assert reconciler.outcome_of(summary) == ("done", "") and summary["reused"] == reused
|
||||
state = json.loads((base / "state.json").read_text())
|
||||
assert state["tasks"]["c-facts"]["status"] == "done" and state["tasks"]["c-facts"]["reused"] == reused
|
||||
assert not (base / "lock").exists()
|
||||
assert seed(manifest, stored, results_root=results) == [], "a seeded task is never seeded again"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change", [{"pin": OTHER_PIN}, {"candidates": _digest(["run", "helper"])}, {"image": "reg/img@sha256:" + "b" * 64}]
|
||||
)
|
||||
def test_seed_recomputes_on_a_changed_pin_candidate_set_or_image(stored: FileStore, tmp_path: Path, change: dict[str, str]) -> None:
|
||||
key = {"repo": "https://github.com/o/r", "pin": PIN, "candidates": _digest(["run"]), "image": f"reg/img@{IMAGE}", **change}
|
||||
manifest = _plane(tmp_path / "plane", memory_key(**key))
|
||||
assert seed(manifest, stored, results_root=tmp_path / "results") == []
|
||||
assert not (tmp_path / "results" / "r2").exists()
|
||||
|
||||
|
||||
def test_seed_never_overwrites_a_task_directory_and_drops_tampered_facts(stored: FileStore, tmp_path: Path) -> None:
|
||||
manifest = _plane(tmp_path / "plane", memory_key("https://github.com/o/r", PIN, _digest(["run"]), f"reg/img@{IMAGE}"))
|
||||
results = tmp_path / "results"
|
||||
(results / "r2" / "c-facts").mkdir(parents=True)
|
||||
assert seed(manifest, stored, results_root=results) == []
|
||||
(results / "r2" / "c-facts").rmdir()
|
||||
sha = hashlib.sha256(FACTS).hexdigest()
|
||||
(stored.root / "facts" / f"{sha}.json").write_bytes(b"changed underneath")
|
||||
assert seed(manifest, stored, results_root=results) == []
|
||||
assert not (stored.root / "facts" / f"{sha}.json").exists()
|
||||
@@ -756,6 +756,22 @@ def test_the_task_is_applied_after_everything_it_binds(tmp_path: Path) -> None:
|
||||
assert docs[-1]["kind"] == "Task", [d["kind"] for d in docs]
|
||||
|
||||
|
||||
def test_mark_done_sets_the_state_under_the_run_lock(tmp_path: Path) -> None:
|
||||
import os
|
||||
|
||||
base = tmp_path / "r"
|
||||
base.mkdir()
|
||||
(base / "lock").write_text(json.dumps(os.getppid())) # a live process that is not this one holds the run
|
||||
with pytest.raises(RuntimeError, match="run is held by pid"):
|
||||
reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path)
|
||||
assert not (base / "state.json").exists()
|
||||
(base / "lock").unlink()
|
||||
reconciler.mark_done("r", "c-facts", reused={"sha256": "ab", "from_run": "r0"}, results_root=tmp_path)
|
||||
task = json.loads((base / "state.json").read_text())["tasks"]["c-facts"]
|
||||
assert (task["status"], task["reused"]) == ("done", {"sha256": "ab", "from_run": "r0"}) and task["finished"]
|
||||
assert not (base / "lock").exists(), "the lock is released"
|
||||
|
||||
|
||||
def test_reconciler_is_under_500_lines_and_holds_no_pipeline_logic() -> None:
|
||||
text = RECONCILER.read_text(encoding="utf-8")
|
||||
assert len(text.splitlines()) < 500, "Req 3.4: the reconciler stays under 500 lines"
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
"""`remember` (harnessx-removal §4) on a recorded Run: a few KB of validation-46's lmdeploy case under
|
||||
``tests/fixtures/plane-remember`` (witness texts cut, facts.json cut to the files its callers name, the repository
|
||||
URL replaced: the population is reserved, ``tests/test_independent_population.py``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.plane import memory
|
||||
from openultrasast.plane.memory import FileStore, MemoryStoreError, ingest, row_id
|
||||
from openultrasast.plane.tasks import remember
|
||||
from openultrasast.plane.tasks.remember import Context, candidates_digest, remember_run, rows_for
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||
CASE = "lmdeploy-media-url-ssrf"
|
||||
PIN = "71648d239c978452720a6d6692fe914efc8a7b94"
|
||||
IMAGE = "sha256:200e335877b52d68ea0bc53b4c9ba49ef2ee19e8a5ec2c874148b969123ef7ab"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "plane" / "validation-46"
|
||||
shutil.copytree(FIXTURE, target)
|
||||
return target
|
||||
|
||||
|
||||
def _by_kind(store: FileStore) -> dict[str, list[dict[str, Any]]]:
|
||||
rows: dict[str, list[dict[str, Any]]] = {}
|
||||
for record in store.rows():
|
||||
rows.setdefault(record.row["kind"], []).append(record.row)
|
||||
return rows
|
||||
|
||||
|
||||
def test_remember_turns_the_recorded_case_into_rows(run_dir: Path, tmp_path: Path) -> None:
|
||||
facts = (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
assert len(facts) > 100, "the fixture was read"
|
||||
store = FileStore(tmp_path / "memory")
|
||||
results = remember_run(run_dir, store, population="population-v2", split="validation")
|
||||
assert [(r.task, r.skipped, r.kinds) for r in results] == [(f"{CASE}-remember", False, {"facts": 1, "unit_cost": 2, "verdict": 1})]
|
||||
rows = _by_kind(store)
|
||||
for row in (r for kind in rows.values() for r in kind):
|
||||
assert (row["repo"], row["pin"], row["run"], row["image"]) == ("example.com/reserved/lmdeploy", PIN, "validation-46", IMAGE)
|
||||
assert (row["task"], row["population"], row["split"]) == (f"{CASE}-remember", "population-v2", "validation")
|
||||
sha = hashlib.sha256(facts).hexdigest()
|
||||
assert rows["facts"][0]["sha256"] == sha and store.get_facts(sha) == facts
|
||||
assert rows["facts"][0]["candidates_digest"] == candidates_digest(json.loads(facts))
|
||||
agreed = json.loads((run_dir / f"{CASE}-final" / "agreed.json").read_text())["candidates"]
|
||||
assert [(v["candidate"], v["passes"], v["site_match"], v["usd"], v["turns"]) for v in rows["verdict"]] == [
|
||||
(c["candidate"], {"a": c["a"], "b": c["b"], "c": c["c"]}, c["site_match"], c["usd"], c["turns"]) for c in agreed
|
||||
], "a verdict row per agree row, from the final (tie-break) decision"
|
||||
assert rows["verdict"][0]["final"] == "agreed" and rows["verdict"][0]["family"] == "untrusted_destination"
|
||||
costs = {(u["pass"], u["path"]): (u["usd"], u["calls"], u["model"]) for u in rows["unit_cost"]}
|
||||
assert costs == {
|
||||
("a", "lmdeploy/vl/media/connection.py"): (0.01237569, 7, "deepseek-flash"),
|
||||
("b", "lmdeploy/vl/media/connection.py"): (costs[("b", "lmdeploy/vl/media/connection.py")][0], 7, "deepseek-flash"),
|
||||
}
|
||||
assert sum(u["usd"] for u in rows["unit_cost"]) == pytest.approx(rows["verdict"][0]["usd"], abs=1e-6)
|
||||
assert not list(run_dir.glob("*-remember")), "the by-hand path writes nothing into the run directory"
|
||||
|
||||
|
||||
def test_re_ingest_of_the_same_run_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
remember_run(run_dir, store)
|
||||
before = {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()}
|
||||
again = remember_run(run_dir, store)
|
||||
assert [r.skipped for r in again] == [True]
|
||||
assert {p: p.read_bytes() for p in store.root.rglob("*") if p.is_file()} == before
|
||||
|
||||
|
||||
def test_rows_carry_every_final_the_tiebreak_and_alerts() -> None:
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
agreed = {
|
||||
"family": "injection",
|
||||
"candidates": [
|
||||
{"candidate": "a.py::f", "a": True, "b": True, "c": None, "agreed": True},
|
||||
{"candidate": "a.py::g", "a": True, "b": False, "c": True, "agreed": True},
|
||||
{"candidate": "a.py::h", "a": True, "b": False, "c": False, "agreed": False},
|
||||
{"candidate": "a.py::i", "a": False, "b": False, "c": None, "agreed": False},
|
||||
],
|
||||
"disputed": [{"candidate": "a.py::h"}],
|
||||
}
|
||||
alert = {"rule_id": "py-ssrf", "rule_status": "shadow", "path": "a.py", "line": 3, "function": "f", "pin_role": "vulnerable"}
|
||||
rows = rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert])
|
||||
verdicts = {r["candidate"]: (r["final"], r["tiebreak"]) for r in rows if r["kind"] == "verdict"}
|
||||
expected = {"a.py::f": ("agreed", False), "a.py::g": ("agreed", True), "a.py::h": ("disputed", True), "a.py::i": ("rejected", False)}
|
||||
assert verdicts == expected
|
||||
(alert_row,) = [r for r in rows if r["kind"] == "alert"]
|
||||
assert {k: alert_row[k] for k in alert} == alert and alert_row["repo"] == "github.com/o/r"
|
||||
assert alert_row["id"] == row_id("alert", "r", "c-remember", "py-ssrf:a.py:3")
|
||||
assert len({r["id"] for r in rows}) == len(rows)
|
||||
assert rows == rows_for(ctx, facts=None, passes={}, models={}, agreed=agreed, alerts=[alert]), "deterministic"
|
||||
|
||||
|
||||
def test_the_task_entrypoint_writes_memory_and_passes_facts_through(run_dir: Path, tmp_path: Path) -> None:
|
||||
docs = list(yaml.safe_load_all((run_dir / f"{CASE}-facts" / "task.yaml").read_text()))
|
||||
task = next(d for d in docs if d["kind"] == "Task")
|
||||
pins = next(e["value"] for e in task["spec"]["env"] if e["name"] == "OUSAST_GIT_PINS")
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_RUN": "validation-46", "OUSAST_TASK": f"{CASE}-remember", "OUSAST_GIT_PINS": pins,
|
||||
"AX_TASK_YAML": yaml.safe_dump(task), "AX_WORKSPACES_YAML": yaml.safe_dump_all([d for d in docs if d["kind"] == "Workspace"]),
|
||||
"OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"),
|
||||
"OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"),
|
||||
"OUSAST_POPULATION": "population-v2", "OUSAST_SPLIT": "validation",
|
||||
**{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"},
|
||||
**{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"},
|
||||
} # fmt: skip
|
||||
assert remember.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["units_done"], summary["usd"], summary["calls"], summary["rows"]) == ("done", 1, 0, 0, 4)
|
||||
assert (out / "facts.json").read_bytes() == (run_dir / f"{CASE}-facts" / "facts.json").read_bytes()
|
||||
delivered = FileStore(tmp_path / "delivered")
|
||||
assert [r.task for r in ingest(run_dir, delivered)] == [f"{CASE}-remember"]
|
||||
by_hand = FileStore(tmp_path / "by-hand")
|
||||
(out / "memory.jsonl").rename(tmp_path / "memory.jsonl")
|
||||
remember_run(run_dir, by_hand, population="population-v2", split="validation")
|
||||
assert [r.row for r in delivered.rows()] == [r.row for r in by_hand.rows()], "the task and the host path agree"
|
||||
assert remember.main({**env, "AX_TASK_YAML": "{}"}) == 2
|
||||
assert "names no image" in json.loads((out / "summary.json").read_text())["reason"]
|
||||
|
||||
|
||||
def test_ingest_without_remember_outputs_is_a_no_op(run_dir: Path, tmp_path: Path) -> None:
|
||||
store = FileStore(tmp_path / "memory")
|
||||
assert ingest(run_dir, store) == [] and not store.root.exists()
|
||||
|
||||
|
||||
def test_a_malformed_delivered_row_fails_naming_file_and_line(run_dir: Path, tmp_path: Path) -> None:
|
||||
out = run_dir / f"{CASE}-remember"
|
||||
out.mkdir()
|
||||
good = rows_for(Context("validation-46", out.name, "https://github.com/o/r", PIN, IMAGE), facts=None, passes={}, models={}, agreed=None,
|
||||
alerts=[{"rule_id": "x", "path": "a.py", "line": 1}]) # fmt: skip
|
||||
(out / "memory.jsonl").write_text(json.dumps(good[0]) + "\n" + json.dumps({**good[0], "kind": "vibes"}) + "\n")
|
||||
store = FileStore(tmp_path / "memory")
|
||||
with pytest.raises(MemoryStoreError, match=rf"{out.name}/memory.jsonl:2: unknown kind"):
|
||||
remember_run(run_dir, store)
|
||||
assert not store.root.exists()
|
||||
|
||||
|
||||
def test_plane_remember_cli_reads_population_from_the_run_and_reports_counts(
|
||||
run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "1 tasks ingested, 0 already in the index; rows facts=1 unit_cost=2 verdict=1" in out
|
||||
rows = FileStore(tmp_path / "memory").rows()
|
||||
assert {(r.row["population"], r.row["split"]) for r in rows} == {("population-v2", "validation")} # plane/runs/validation-46.yaml
|
||||
assert cli.main(["plane", "remember", "validation-46"]) == 0
|
||||
assert "0 tasks ingested, 1 already in the index" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_plane_run_seeds_before_and_ingests_after(run_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.plane import reconciler
|
||||
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path))
|
||||
monkeypatch.setenv("OUSAST_MEMORY", f"file://{tmp_path}/memory")
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(memory, "seed", lambda manifest, store: calls.append(f"seed {manifest.name}") or [])
|
||||
monkeypatch.setattr(reconciler, "run", lambda manifest, **kw: calls.append("run") or "done")
|
||||
assert cli.main(["plane", "run", "plane/runs/validation-46.yaml"]) == 0
|
||||
assert calls == ["seed validation-46.yaml", "run"]
|
||||
assert len(FileStore(tmp_path / "memory").rows(kind="verdict")) == 1, "the run's rows were ingested after it"
|
||||
Reference in New Issue
Block a user