mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
engine: worker Deployment from the image (S3 queue with leases, dispatcher --executor queue, engine-image workflow, scoped RustFS policy); merge main
Implemented with Codex. Cluster deploy pending the k3s kubeconfig and the scoped account. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
@@ -26,6 +26,7 @@ from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from pathlib import Path
|
||||
|
||||
from engine_trace_k8s import IMAGE, Kubernetes
|
||||
from engine_trace_queue import Dispatcher
|
||||
from engine_trace_worker import EXCLUDED_DIRS, safe_path, unit_record, write_json
|
||||
|
||||
from openultrasast.learn.examples import read_jsonl
|
||||
@@ -44,7 +45,9 @@ SNAPSHOT_FILES = (
|
||||
"benchmarks/learn/engine_trace.py",
|
||||
"benchmarks/learn/engine_trace_worker.py",
|
||||
"benchmarks/learn/engine_trace_k8s.py",
|
||||
"src/openultrasast/model/trace.py",
|
||||
"benchmarks/learn/engine_trace_queue.py",
|
||||
"benchmarks/learn/engine_worker_service.py",
|
||||
"benchmarks/learn/engine_trace_parse.py",
|
||||
"src/openultrasast/model/taint.py",
|
||||
"src/openultrasast/cpg/queries/taint.sc",
|
||||
)
|
||||
@@ -628,11 +631,14 @@ def run_pin(pin, source, provenance, inputs, out, args):
|
||||
write_json(output / "pin.json", {**pin, "questions": prepare_questions(pin, checkout, args.question_deadline)})
|
||||
name = "ousast-trace-" + pin_name(pin)[:16]
|
||||
started = time.monotonic()
|
||||
if executor == "k8s":
|
||||
if executor in {"k8s", "k8s-jobs"}:
|
||||
done, node = args.kubernetes.execute(source, checkout, output, json.loads((output / "pin.json").read_text()))
|
||||
status, reason = "failed", done.stderr
|
||||
if done.returncode:
|
||||
(output / "container.log").write_text(reason)
|
||||
elif executor == "queue":
|
||||
done, node = args.dispatcher.execute(checkout, output, json.loads((output / "pin.json").read_text()), pin_name(pin)[:-5])
|
||||
status, reason = "timeout", f"queue wait deadline {args.queue_timeout}s"
|
||||
else:
|
||||
command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
|
||||
started = time.monotonic()
|
||||
@@ -666,12 +672,20 @@ def run_pin(pin, source, provenance, inputs, out, args):
|
||||
status=status if unit["supported"] else "unsupported", reason=reason if unit["supported"] else unit["reason"]
|
||||
)
|
||||
record["units"].append(row)
|
||||
record["container_exit"] = done.returncode if done else None
|
||||
if executor != "queue":
|
||||
record["container_exit"] = done.returncode if done else None
|
||||
else:
|
||||
record.setdefault("container_exit", None)
|
||||
if done is not None and done.returncode != 0:
|
||||
for row in record["units"]:
|
||||
if row["supported"]:
|
||||
row.update(status="failed", reason=reason)
|
||||
record.update(done=True, seconds=time.monotonic() - started, analyzer=provenance, image=args.image)
|
||||
record.update(
|
||||
done=True,
|
||||
seconds=time.monotonic() - started,
|
||||
analyzer=provenance,
|
||||
image=record.get("image", args.image) if executor == "queue" else args.image,
|
||||
)
|
||||
if (output / "container.log").exists():
|
||||
shutil.copyfile(output / "container.log", out / pin_name(pin).replace(".json", ".log"))
|
||||
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
||||
@@ -681,8 +695,10 @@ def run_pin(pin, source, provenance, inputs, out, args):
|
||||
"analyzer": provenance,
|
||||
"units": [unit_record(u, "failed" if u["supported"] else "unsupported", str(exc)) for u in pin["units"]],
|
||||
}
|
||||
if executor == "k8s":
|
||||
if executor in {"k8s", "k8s-jobs"}:
|
||||
record.update(executor="k8s", node=node, image=args.image)
|
||||
if executor == "queue":
|
||||
record.update(executor="queue", worker=record.get("worker", node), pod=record.get("pod"))
|
||||
save_pin(out / pin_name(pin), record)
|
||||
print(
|
||||
json.dumps({"repo": pin["repo"], "pin": pin["pin"], "statuses": dict(Counter(u["status"] for u in record["units"]))}),
|
||||
@@ -810,7 +826,9 @@ def main():
|
||||
parser.add_argument("--cache", type=Path, default=Path.home() / ".cache/openultrasast")
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
parser.add_argument("--image", help="default: openultrasast:dev for Docker; public 2.0.1 image for k8s/mixed")
|
||||
parser.add_argument("--executor", choices=("docker", "k8s", "mixed"), default="docker")
|
||||
parser.add_argument("--executor", choices=("docker", "k8s", "k8s-jobs", "mixed", "queue"), default="docker")
|
||||
parser.add_argument("--queue-status", action="store_true")
|
||||
parser.add_argument("--queue-timeout", type=int, default=3600, help="total queue wait including execution, seconds")
|
||||
parser.add_argument("--parallel", type=int, default=1, help="total pins in flight; mixed reserves one lane for Docker")
|
||||
parser.add_argument("--kube-context")
|
||||
parser.add_argument("--kube-namespace", default="ousast-engine")
|
||||
@@ -829,11 +847,20 @@ def main():
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--summary", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.queue_status:
|
||||
print(json.dumps(Dispatcher(args).status(), indent=2))
|
||||
return 0
|
||||
if args.queue_timeout <= 0:
|
||||
parser.error("--queue-timeout must be positive")
|
||||
if args.executor == "queue" and (not args.image or "@sha256:" not in args.image):
|
||||
parser.error("queue requires --image with the deployed image digest")
|
||||
if args.executor == "queue" and args.deadline > 900:
|
||||
parser.error("queue deadline must be <= Deployment termination grace (900s)")
|
||||
if min(args.deadline, args.question_deadline) <= 0 or args.limit < 0:
|
||||
parser.error("deadlines must be positive; limit must be nonnegative")
|
||||
if args.parallel < 1 or (args.executor == "mixed" and args.parallel < 2):
|
||||
parser.error("--parallel must be positive; mixed needs at least 2 total lanes")
|
||||
if (args.executor != "docker" or args.k8s_check) and not args.kube_context:
|
||||
if (args.executor in {"k8s", "k8s-jobs", "mixed"} or args.k8s_check) and not args.kube_context:
|
||||
parser.error("--kube-context is required for Kubernetes")
|
||||
args.image = args.image or (IMAGE if args.executor != "docker" or args.k8s_check else "openultrasast:dev")
|
||||
if args.k8s_check:
|
||||
@@ -905,12 +932,14 @@ def main():
|
||||
args.docker_image = image.stdout.strip()
|
||||
if not args.docker_image:
|
||||
raise ValueError("Docker returned no local image identity")
|
||||
if args.executor != "docker" and any(u["supported"] for pin in plan for u in pin["units"]):
|
||||
if args.executor in {"k8s", "k8s-jobs", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
|
||||
args.kubernetes = Kubernetes(args)
|
||||
args.kubernetes.namespace()
|
||||
if args.executor == "queue":
|
||||
args.dispatcher = Dispatcher(args)
|
||||
with tempfile.TemporaryDirectory(prefix="trace-source-", dir=args.out) as scratch:
|
||||
source = Path(scratch) / "frozen"
|
||||
provenance = freeze_source(source)
|
||||
provenance = {"image": args.image, "executor": "queue"} if args.executor == "queue" else freeze_source(source)
|
||||
failures = run_plan(plan, source, provenance, inputs, args)
|
||||
return 2 if failures else 0
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Lossless parsing of the optional shortest-path trace, with explicit evidence limits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
_STEP = re.compile(r"^(.*):(-?\d+) \[([^\]]+)\] ([\s\S]*)$")
|
||||
|
||||
|
||||
def parse_trace(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Preserve row-level discharge evidence; never guess which node sanitized a path.
|
||||
|
||||
The query emits basename-only files and truncated code. A bound can be off-path;
|
||||
it remains in ``guard`` even when no trace step exactly matches it. Invalid steps
|
||||
raise instead of turning an unreadable trace into an empty path.
|
||||
"""
|
||||
raw = row.get("trace", [])
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("trace must be an array")
|
||||
steps = []
|
||||
bound = str(row.get("bound") or "")
|
||||
for index, text in enumerate(raw):
|
||||
match = _STEP.fullmatch(text) if isinstance(text, str) else None
|
||||
if match is None:
|
||||
raise ValueError(f"invalid trace step: {text!r}")
|
||||
location, line, node_label, code = match.groups()
|
||||
file, separator, method = location.partition(":")
|
||||
roles = ["source"] if index == 0 else ["step"]
|
||||
if index == len(raw) - 1:
|
||||
roles = ["source", "sink"] if index == 0 else ["sink"]
|
||||
if bound and code == bound:
|
||||
roles.append("guard")
|
||||
steps.append(
|
||||
{
|
||||
"file": file,
|
||||
"method": method if separator else "?",
|
||||
"line": int(line),
|
||||
"node_label": node_label,
|
||||
"code": code,
|
||||
"roles": roles,
|
||||
"raw": text,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"steps": steps,
|
||||
"sanitized": bool(row.get("sanitized", False)),
|
||||
"sanitizer": {"on_path": bool(row.get("sanitized", False)), "step": None},
|
||||
"guard": {"text": bound, "bounded": bool(row.get("bounded", False))},
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Host queue transport: source and task metadata only, never analyzer code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from engine_worker_service import PENDING, client_from_env, pack, put, read, result_key, status
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
def __init__(self, args, client=None, *, pause=time.sleep, clock=time.monotonic):
|
||||
self.client = client if client is not None else client_from_env()
|
||||
self.run = uuid.uuid4().hex
|
||||
self.args, self.pause, self.clock = args, pause, clock
|
||||
|
||||
def execute(self, checkout, output, pin, pin_id):
|
||||
task_id = f"{self.run}-{pin_id}"
|
||||
task = dict(
|
||||
run=self.run,
|
||||
id=pin_id,
|
||||
task_id=task_id,
|
||||
pin=pin,
|
||||
input=f"engine-queue/inputs/{self.run}/{pin_id}/source.tar",
|
||||
image=self.args.image,
|
||||
deadline=self.args.deadline,
|
||||
question_deadline=self.args.question_deadline,
|
||||
)
|
||||
try:
|
||||
self.client.put(task["input"], pack(checkout), {})
|
||||
put(self.client, PENDING + task_id, task)
|
||||
end = self.clock() + self.args.queue_timeout
|
||||
while self.clock() < end:
|
||||
result = read(self.client, result_key(task))
|
||||
if result is not None:
|
||||
(output / "result.json").write_text(json.dumps(result))
|
||||
self.client.delete(task["input"])
|
||||
return subprocess.CompletedProcess([], 0, "", ""), result.get("worker")
|
||||
self.pause(2)
|
||||
# Removing pending cancels work that has not been claimed; retain source for a live lease.
|
||||
self.client.delete(PENDING + task_id)
|
||||
return None, None
|
||||
except Exception as exc:
|
||||
# SDK errors can contain endpoint URLs or credentials. Do not persist their text.
|
||||
raise OSError("queue transport failed") from exc
|
||||
|
||||
def status(self):
|
||||
return status(self.client)
|
||||
@@ -19,8 +19,9 @@ import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from engine_trace_parse import parse_trace
|
||||
|
||||
from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, _render
|
||||
from openultrasast.model.trace import parse_trace
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Image-resident queue workers. Requires strongly consistent S3 GET/PUT/LIST.
|
||||
|
||||
Per-task Lamport bakery tickets use only ordinary object operations. A choosing
|
||||
marker precedes ticket selection; contenders wait for choosing peers and lower
|
||||
(ticket, worker) pairs. Late arrivals cannot displace a running owner. Leases
|
||||
assume bounded clock skew (less than MARGIN / 2) and a worker killed by its deadline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import ctypes
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from engine_trace_worker import unit_record, worker
|
||||
|
||||
from openultrasast.plane.memory import S3Client, s3_settings
|
||||
|
||||
PENDING = "engine-queue/pending/"
|
||||
CLAIMED = "engine-queue/claimed/"
|
||||
RESULTS = "engine-results/"
|
||||
MARGIN = 60
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def client_from_env():
|
||||
settings = s3_settings(os.environ)
|
||||
bucket = os.environ.get("S3_BUCKET")
|
||||
if not bucket:
|
||||
raise ValueError("S3_BUCKET is required")
|
||||
return S3Client(bucket=bucket, **settings)
|
||||
|
||||
|
||||
def read(client, key):
|
||||
value = client.get(key)
|
||||
return json.loads(value[0]) if value else None
|
||||
|
||||
|
||||
def put(client, key, value):
|
||||
client.put(key, json.dumps(value).encode(), {})
|
||||
|
||||
|
||||
def result_key(task):
|
||||
return f"{RESULTS}{task['run']}/{task['id']}.json"
|
||||
|
||||
|
||||
def pack(root):
|
||||
stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=stream, mode="w") as archive:
|
||||
archive.add(root, arcname=".")
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def unpack(data, root):
|
||||
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
||||
# Source snapshots need only regular files and directories, never links/devices.
|
||||
for member in archive.getmembers():
|
||||
target = (root / member.name).resolve()
|
||||
if not target.is_relative_to(root.resolve()) or not (member.isfile() or member.isdir()):
|
||||
raise ValueError("unsafe source archive")
|
||||
archive.extractall(root, filter="data")
|
||||
|
||||
|
||||
class Queue:
|
||||
def __init__(self, client, identity=None, *, now=time.time, pause=time.sleep, stop=None):
|
||||
self.client = client
|
||||
self.identity = identity or f"{socket.gethostname()}-{uuid.uuid4().hex}"
|
||||
self.now, self.pause = now, pause
|
||||
self.stop = stop or threading.Event()
|
||||
|
||||
def markers(self, task_id=None):
|
||||
result = []
|
||||
for key in self.client.keys(CLAIMED):
|
||||
if task_id is None or key.rsplit("/", 1)[-1] == task_id:
|
||||
marker = read(self.client, key)
|
||||
if marker:
|
||||
result.append((key, marker))
|
||||
return result
|
||||
|
||||
def tasks(self):
|
||||
tasks = {}
|
||||
for key in self.client.keys(PENDING):
|
||||
task = read(self.client, key)
|
||||
if task:
|
||||
tasks[task["task_id"]] = task
|
||||
# A dead worker may have removed pending already. Its task remains in the lease.
|
||||
for _, marker in self.markers():
|
||||
if marker["expires"] < self.now():
|
||||
task = marker["task"]
|
||||
tasks.setdefault(task["task_id"], task)
|
||||
return list(tasks.values())
|
||||
|
||||
def claim(self, task):
|
||||
if read(self.client, result_key(task)) is not None or self.stop.is_set():
|
||||
return None
|
||||
key = f"{CLAIMED}{self.identity}/{task['task_id']}"
|
||||
marker = dict(task=task, ticket=0, expires=self.now() + task["deadline"] + MARGIN)
|
||||
put(self.client, key, marker)
|
||||
try:
|
||||
live = [m for _, m in self.markers(task["task_id"]) if m["expires"] > self.now()]
|
||||
marker["ticket"] = 1 + max((m["ticket"] for m in live), default=0)
|
||||
put(self.client, key, marker)
|
||||
while not self.stop.is_set():
|
||||
# Do not start once the execution budget no longer fits within our lease.
|
||||
if self.now() >= marker["expires"] - task["deadline"] - MARGIN / 2:
|
||||
break
|
||||
peers = [(k, m) for k, m in self.markers(task["task_id"]) if k != key and m["expires"] > self.now()]
|
||||
if any(m["ticket"] == 0 for _, m in peers):
|
||||
self.pause(0.1)
|
||||
continue
|
||||
if any((m["ticket"], k) < (marker["ticket"], key) for k, m in peers):
|
||||
break # loser backs off and can try another pending task
|
||||
if read(self.client, key) != marker or read(self.client, result_key(task)) is not None:
|
||||
break
|
||||
self.client.delete(PENDING + task["task_id"])
|
||||
return key
|
||||
except Exception:
|
||||
self.client.delete(key)
|
||||
raise
|
||||
self.client.delete(key)
|
||||
return None
|
||||
|
||||
def process_one(self, execute=None):
|
||||
for task in self.tasks():
|
||||
key = self.claim(task)
|
||||
if key is None:
|
||||
continue
|
||||
try:
|
||||
lease = read(self.client, key)
|
||||
if lease is None:
|
||||
continue
|
||||
budget = min(task["deadline"], lease["expires"] - self.now() - MARGIN / 2)
|
||||
if budget <= 0:
|
||||
continue # Leave the expired lease's task available to recovery.
|
||||
result = (execute or execute_task)(self.client, {**task, "runtime_budget": budget})
|
||||
result.update(executor="queue", worker=self.identity, pod=socket.gethostname(), image=os.environ.get("ENGINE_IMAGE", ""))
|
||||
put(self.client, result_key(task), result)
|
||||
# Publish the result before removing either the input or the claim.
|
||||
self.client.delete(task["input"])
|
||||
for old_key, old in self.markers(task["task_id"]):
|
||||
if old_key == key or old["expires"] < self.now():
|
||||
self.client.delete(old_key)
|
||||
return True
|
||||
except Exception:
|
||||
# Keep the lease and task for recovery, without logging credential-bearing errors.
|
||||
LOG.error("task failed; lease retained for recovery")
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _child(pin, root, out, deadline, question_deadline):
|
||||
os.setsid()
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
worker(pin, root, out, deadline, question_deadline)
|
||||
|
||||
|
||||
def adopt_orphans():
|
||||
# Linux PR_SET_CHILD_SUBREAPER: escaped Joern sessions reparent here, not
|
||||
# outside the service. There is exactly one task process tree per worker.
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError("cannot enable task descendant cleanup")
|
||||
|
||||
|
||||
def kill_children():
|
||||
"""Freeze parents before walking children, including new-session JVMs."""
|
||||
seen = set()
|
||||
|
||||
def freeze(parent):
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdigit() or int(entry.name) in seen:
|
||||
continue
|
||||
try:
|
||||
# comm can contain spaces or parentheses. Fields after it begin at state.
|
||||
fields = (entry / "stat").read_text().rsplit(")", 1)[1].split()
|
||||
if int(fields[1]) != parent:
|
||||
continue
|
||||
pid = int(entry.name)
|
||||
os.kill(pid, signal.SIGSTOP)
|
||||
seen.add(pid)
|
||||
freeze(pid)
|
||||
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
||||
continue
|
||||
|
||||
# A child that exited during discovery can reparent descendants to this
|
||||
# subreaper. Repeat until every remaining descendant is frozen.
|
||||
while True:
|
||||
count = len(seen)
|
||||
freeze(os.getpid())
|
||||
if len(seen) == count:
|
||||
break
|
||||
for pid in seen:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
|
||||
|
||||
def execute_task(client, task):
|
||||
adopt_orphans()
|
||||
pin = task["pin"]
|
||||
started = time.monotonic()
|
||||
budget = task.get("runtime_budget", task["deadline"])
|
||||
status, reason, code = "failed", "worker produced no result", None
|
||||
with tempfile.TemporaryDirectory(prefix="engine-task-") as scratch:
|
||||
root, out = Path(scratch) / "case", Path(scratch) / "result.json"
|
||||
root.mkdir()
|
||||
try:
|
||||
if task["deadline"] > 900 or task["deadline"] <= 0:
|
||||
raise ValueError("deadline exceeds deployment grace")
|
||||
if task.get("image") != os.environ.get("ENGINE_IMAGE"):
|
||||
raise ValueError("task image differs from deployed image")
|
||||
source = client.get(task["input"])
|
||||
if source is None:
|
||||
raise ValueError("missing source")
|
||||
unpack(source[0], root)
|
||||
# The shipped worker also checks frontend input and graph output for every language.
|
||||
files = [p for p in root.rglob("*") if p.is_file()]
|
||||
if not files:
|
||||
raise ValueError("empty source")
|
||||
LOG.info("input proof: first file bytes=%d", len(files[0].read_bytes()))
|
||||
if time.monotonic() - started >= budget:
|
||||
status, reason = "timeout", "task input deadline"
|
||||
raise TimeoutError(reason)
|
||||
process = multiprocessing.get_context("fork").Process(target=_child, args=(pin, root, out, budget, task["question_deadline"]))
|
||||
process.start()
|
||||
try:
|
||||
process.join(max(0, budget - (time.monotonic() - started)))
|
||||
if process.is_alive():
|
||||
status, reason = "timeout", "task deadline"
|
||||
else:
|
||||
reason = f"worker exit {process.exitcode}"
|
||||
code = process.exitcode
|
||||
finally:
|
||||
# Joern creates separate sessions. Kill the whole descendant tree,
|
||||
# not merely the Python wrapper's process group.
|
||||
kill_children()
|
||||
process.join()
|
||||
# Reap adopted JVMs and their launchers after the direct child.
|
||||
while True:
|
||||
try:
|
||||
os.waitpid(-1, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
record = json.loads(out.read_text()) if out.exists() else {**pin, "units": []}
|
||||
except Exception:
|
||||
record = {**pin, "units": []}
|
||||
reason = "worker input or execution failed"
|
||||
if not record.get("done") or code != 0:
|
||||
previous = {u["unit"]: u for u in record["units"]}
|
||||
record["units"] = [
|
||||
previous[u["unit"]]
|
||||
if code is None and previous.get(u["unit"], {}).get("status") in {"path", "asked-nothing"}
|
||||
else unit_record(u, status if u["supported"] else "unsupported", reason if u["supported"] else u["reason"])
|
||||
for u in pin["units"]
|
||||
]
|
||||
record.update(done=True, container_exit=code, seconds=time.monotonic() - started)
|
||||
return record
|
||||
|
||||
|
||||
def status(client):
|
||||
return {
|
||||
"pending": len(client.keys(PENDING)),
|
||||
"claimed": len(client.keys(CLAIMED)),
|
||||
"done": len([k for k in client.keys(RESULTS) if k.endswith(".json")]),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
stop = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: stop.set())
|
||||
client = client_from_env()
|
||||
queue = Queue(client, stop=stop)
|
||||
|
||||
class Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200 if self.path == "/healthz" and not stop.is_set() else 503)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("0.0.0.0", args.port), Health)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
if not queue.process_one():
|
||||
stop.wait(2)
|
||||
except Exception:
|
||||
LOG.error("queue unavailable; retrying")
|
||||
stop.wait(5)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Offline audit of completed Joern results; never launches an analyzer or reads corpora.
|
||||
|
||||
Missing results stay in coverage, not signal denominators. Observed non-path rows
|
||||
score zero for path features. Sink kinds are one-hot indicators, never ordinal
|
||||
category IDs. Both signed and best-direction metrics expose reversed shortcuts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from slice_leak_audit import THRESHOLD, complete_pairs, metrics
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
UNITS = ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl"
|
||||
RESULTS = Path.home() / "ousast-results/plane/engine-trace"
|
||||
OUTPUT = ROOT / "benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json"
|
||||
STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported", "missing")
|
||||
ASKED = {"path", "asked-nothing"}
|
||||
SIGNALS = ("has_path", "shortest_path_steps", "sanitized", "guard_present", "asked")
|
||||
ASYMMETRY = ("vulnerable_only", "fixed_only", "both", "neither", "not_both_asked")
|
||||
|
||||
|
||||
def materialization_version():
|
||||
# Read the producer's literal without importing its execution dependencies.
|
||||
tree = ast.parse(Path(__file__).with_name("engine_trace.py").read_bytes())
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "MATERIALIZATION_VERSION" for t in node.targets):
|
||||
return ast.literal_eval(node.value)
|
||||
raise ValueError("Producer has no materialization version")
|
||||
|
||||
|
||||
def features(row):
|
||||
path = row["status"] == "path"
|
||||
traces = row.get("traces", []) if path else []
|
||||
candidates = [(i, t) for i, t in enumerate(traces) if t.get("steps")]
|
||||
index, trace = min(candidates, key=lambda it: len(it[1]["steps"])) if candidates else (None, {})
|
||||
witnesses = row.get("witness_rows", [])
|
||||
witness = witnesses[index] if index is not None and index < len(witnesses) else {}
|
||||
guard = trace.get("guard", {})
|
||||
sink = witness.get("sinkKind")
|
||||
if not sink:
|
||||
# Retain only the call target: arguments, literals and code are not kinds.
|
||||
match = re.search(r"([\w:$>.\\-]+)\s*\(", str(witness.get("sink", "")))
|
||||
sink = match[1] if match else "unknown"
|
||||
return {
|
||||
"has_path": int(path),
|
||||
"shortest_path_steps": len(trace["steps"]) if trace else None if path else 0,
|
||||
"sanitized": int(bool(trace.get("sanitized", witness.get("sanitized", False)))) if trace else None if path else 0,
|
||||
"guard_present": int(
|
||||
bool(guard.get("text") or guard.get("bounded") or witness.get("bound") or witness.get("bounded"))
|
||||
or any("guard" in s.get("roles", []) for s in trace.get("steps", []))
|
||||
)
|
||||
if trace
|
||||
else None
|
||||
if path
|
||||
else 0,
|
||||
"sink_kind": str(sink) if path else "no-path",
|
||||
"asked": int(row["status"] in ASKED),
|
||||
}
|
||||
|
||||
|
||||
def scored(rows, signal):
|
||||
eligible = [r for r in rows if r[signal] is not None]
|
||||
result = metrics(eligible, signal)
|
||||
result["excluded_rows"] = len(rows) - len(eligible)
|
||||
result["flagged"] = (result["pairs"] >= 10 and (result["best_direction_within_pair_accuracy"] or 0) >= THRESHOLD) or (
|
||||
result["rows"] >= 10 and (result["best_direction_pooled_auc"] or 0) >= THRESHOLD
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def signal_metrics(rows, kinds):
|
||||
return {
|
||||
**{s: scored(rows, s) for s in SIGNALS},
|
||||
"sink_kind": {kind: scored([{**r, "is_kind": int(r["sink_kind"] == kind)} for r in rows], "is_kind") for kind in kinds},
|
||||
}
|
||||
|
||||
|
||||
def scope_report(rows, kinds):
|
||||
pairs = complete_pairs(rows)
|
||||
asymmetry = Counter()
|
||||
honest = []
|
||||
for v, f in pairs:
|
||||
if not (v["asked"] and f["asked"]):
|
||||
asymmetry["not_both_asked"] += 1
|
||||
else:
|
||||
honest.extend((v, f))
|
||||
asymmetry[
|
||||
"both"
|
||||
if v["has_path"] and f["has_path"]
|
||||
else "vulnerable_only"
|
||||
if v["has_path"]
|
||||
else "fixed_only"
|
||||
if f["has_path"]
|
||||
else "neither"
|
||||
] += 1
|
||||
observed = [r for r in rows if r["status"] != "missing"]
|
||||
return {
|
||||
"rows": len(rows),
|
||||
"coverage": {s: sum(r["status"] == s for r in rows) for s in STATUSES},
|
||||
"complete_pairs": len(pairs),
|
||||
"pair_asymmetry": {s: asymmetry[s] for s in ASYMMETRY},
|
||||
"both_asked_pairs": len(honest) // 2,
|
||||
"signals": signal_metrics(observed, kinds),
|
||||
"both_asked_signals": signal_metrics(honest, kinds),
|
||||
}
|
||||
|
||||
|
||||
def audit(units, results, *, expected_rows=501, expected_pairs=195):
|
||||
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
|
||||
raise ValueError(f"Expected {expected_rows} unique units")
|
||||
if len(complete_pairs(units)) != expected_pairs:
|
||||
raise ValueError(f"Expected {expected_pairs} complete pairs")
|
||||
if not results.is_dir():
|
||||
raise ValueError(f"Results directory is unreadable: {results}")
|
||||
version = materialization_version()
|
||||
unit_index = {u["unit"]: u for u in units}
|
||||
matched = {}
|
||||
files = []
|
||||
counts = Counter()
|
||||
for path in sorted(results.glob("*.json")):
|
||||
raw = path.read_bytes()
|
||||
record = json.loads(raw) # Corruption fails loudly; producer uses atomic replace.
|
||||
if not isinstance(record, dict) or "repo" not in record or "pin" not in record:
|
||||
continue
|
||||
stale = record.get("materialization_version") != version
|
||||
unfinished = record.get("done") is not True
|
||||
counts["stale"] += stale
|
||||
counts["unfinished"] += unfinished
|
||||
counts["skipped"] += stale or unfinished
|
||||
files.append(
|
||||
{"path": str(path), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), "stale": stale, "unfinished": unfinished}
|
||||
)
|
||||
if stale or unfinished:
|
||||
continue
|
||||
counts["accepted_files"] += 1
|
||||
counts["result_rows_read"] += len(record["units"])
|
||||
for row in record["units"]:
|
||||
if row["unit"] not in unit_index:
|
||||
continue
|
||||
unit = unit_index[row["unit"]]
|
||||
if any(row.get(k) != unit.get(k) for k in ("family", "label", "pair")):
|
||||
raise ValueError(f"Result metadata mismatch: {row['unit']}")
|
||||
if row["status"] not in STATUSES[:-1] or not row.get("source"):
|
||||
raise ValueError(f"Invalid result status/source: {row['unit']}")
|
||||
if row["unit"] in matched:
|
||||
raise ValueError(f"Duplicate current result: {row['unit']}")
|
||||
matched[row["unit"]] = {
|
||||
**unit,
|
||||
"source": row["source"],
|
||||
"status": row["status"],
|
||||
"repo": record["repo"],
|
||||
"pin": record["pin"],
|
||||
"pin_role": row.get("pin_role"),
|
||||
**features(row),
|
||||
}
|
||||
rows = [
|
||||
matched.get(u["unit"], {**u, "source": u.get("source", "unknown"), "status": "missing", **features({"status": "missing"})})
|
||||
for u in units
|
||||
]
|
||||
kinds = sorted({r["sink_kind"] for r in rows if r["status"] != "missing"})
|
||||
scopes = {"overall": scope_report(rows, kinds)}
|
||||
for field in ("family", "source"):
|
||||
scopes[f"by_{field}"] = {
|
||||
value: scope_report([r for r in rows if r[field] == value], kinds) for value in sorted({r[field] for r in rows})
|
||||
}
|
||||
return {
|
||||
"cost_usd": 0,
|
||||
"materialization_version": version,
|
||||
"threshold": THRESHOLD,
|
||||
"minimum_samples": 10,
|
||||
"unit_rows_read": len(units),
|
||||
"matched_rows": len(matched),
|
||||
"inputs": {
|
||||
**{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
|
||||
"files_read": len(files),
|
||||
"bytes_read": sum(f["bytes"] for f in files),
|
||||
"files": files,
|
||||
},
|
||||
"semantics": {
|
||||
"signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
|
||||
"missing": "Coverage only; absent source metadata is unknown. Skipped result units are never consumed.",
|
||||
"non_path": "Observed non-path rows score zero; path rows lacking trace evidence have null trace features.",
|
||||
"asked": "path or asked-nothing=1; failed, timeout, unsupported=0; missing excluded from metrics.",
|
||||
"asymmetry": "Disjoint categories; not_both_asked takes precedence over path asymmetry.",
|
||||
"flags": "Best of both directions, >=0.70 with >=10 pairs for pair accuracy or >=10 rows for AUC.",
|
||||
"sink_kind": "One binary indicator per observed call target, including no-path/unknown; no ordinal encoding.",
|
||||
"limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
|
||||
},
|
||||
**scopes,
|
||||
"units": rows,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--results", type=Path, default=RESULTS)
|
||||
parser.add_argument("--units", type=Path, default=UNITS)
|
||||
parser.add_argument("--output", type=Path, default=OUTPUT)
|
||||
args = parser.parse_args()
|
||||
raw = args.units.read_bytes()
|
||||
units = [json.loads(line) for line in raw.splitlines() if line.strip()]
|
||||
report = audit(units, args.results)
|
||||
report["units_input"] = {"path": str(args.units), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
overall, inputs = report["overall"], report["inputs"]
|
||||
lines = [
|
||||
f"Joern leak audit: $0; materialization_version={report['materialization_version']}",
|
||||
f"Units read: {len(units)}; bytes={len(raw)}; file={args.units}",
|
||||
f"Result files read: {inputs['files_read']}; bytes={inputs['bytes_read']}; accepted={inputs['accepted_files']}",
|
||||
f"Skipped: {inputs['skipped']} (stale={inputs['stale']}, unfinished={inputs['unfinished']}; counts overlap)",
|
||||
f"Result rows read: {inputs['result_rows_read']}; matched={report['matched_rows']}",
|
||||
f"Coverage: {overall['coverage']}",
|
||||
f"Pairs: {overall['complete_pairs']}; both asked={overall['both_asked_pairs']}",
|
||||
f"Asymmetry: {overall['pair_asymmetry']}",
|
||||
]
|
||||
for signal in (*SIGNALS, "sink_kind"):
|
||||
|
||||
def brief(value, signal=signal):
|
||||
if signal == "sink_kind":
|
||||
return {k: brief_metric(m) for k, m in value.items()}
|
||||
return brief_metric(value)
|
||||
|
||||
lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
|
||||
lines.append(f"Family/source metrics and per-file read evidence: {args.output}")
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
def brief_metric(metric):
|
||||
return {k: metric[k] for k in ("rows", "pairs", "within_pair_accuracy", "pooled_auc", "flagged")}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,326 @@
|
||||
{
|
||||
"created": "2026-10-04",
|
||||
"note": "COUNTS ONLY. Batch 3 of real (vulnerable function, fixed function) pairs whose function survives the fix, mined at zero model cost from reviewed GitHub advisories (pip, npm, maven, composer) and added as pointer-pair corpus benchmarks/pairs/advisory-fixes-3 (source `advisory-fixes-3` in learn/sources.toml); labelled, given static feature records, built into the memory store and embedded. No chat-model call, no Docker, no Joern. Candidate, path, function and repository names stay in the catalog; the pairs the reserved-population guard flagged are withheld (found by bisection with the test output discarded; names never printed).",
|
||||
"method": {
|
||||
"enumeration": "gh api /advisories?type=reviewed&ecosystem=<eco>&cwes=<family CWEs>&published=>2023-01-01 per ecosystem (pip, npm, maven, composer) x six compiled families; Go, Ruby and C# not enumerated; newest first",
|
||||
"advisories_enumerated": 8531,
|
||||
"advisories_published_after_2026_06_01": 1855,
|
||||
"automated_screen": "as batch 2 (single family by CWE; linked fix commit, at most 3, one repository; repository not named anywhere under benchmarks/ incl. batches 1-2; excluded v3 file names never opened; no WordPress plugin; one pair per repository; SPDX licence permissive; one-parent commit; the function holding most changed code lines inside its own matcher span, declared once on both sides, body changed, >= 25% of the file's changed code lines) plus three widenings: (a) an advisory linking only a merged pull request uses its single commit or its squash/rebase merge commit; (b) commits of up to 6 non-test source files / 240 changed lines (batch 2: 4 / 120); (c) a second pass over advisories whose CWEs span two families, the family then chosen from the function's diff on review",
|
||||
"file_reads": "commit metadata and the two versions of each changed source file through the contents API into scratch (no clone, no scanner, engine, plane Run or model on these repositories); scratch deleted after",
|
||||
"manual_review": "every automated candidate's in-function diff read (agent, no model call); kept only when the named function itself holds the fix; a weak inline check replaced by a stronger helper call counts as the fix, a function that only forwards a flag or calls a fix made elsewhere does not",
|
||||
"reserved_population_guard": "pytest -q tests/ -k 'reserved or v3' with output discarded, bisected over catalog subsets (18 runs; baseline with an empty batch green); final catalog green",
|
||||
"rename_check": "canonical full_name of every kept repository from the API: no two kept pairs share a repository, none renamed into a name the benchmarks use",
|
||||
"mechanism": "family default as batch 2 (injection/output_encoding/deserialization source_reaches_sink, path path_join_user_input, access_control missing_auth_guard, untrusted_destination validation_strength for CWE-601 else source_reaches_sink); provenance = agent"
|
||||
},
|
||||
"screened": {
|
||||
"advisories_examined": 8805,
|
||||
"outcomes": {
|
||||
"no fix commit linked": 3480,
|
||||
"repository already in benchmarks/ (populations v1/v2 or a pair corpus)": 1993,
|
||||
"licence not permissive or unrecorded": 1249,
|
||||
"repository already kept (one pair per repository)": 657,
|
||||
"kept": 398,
|
||||
"CWEs span several families": 274,
|
||||
"no function declared on both sides with a changed body": 191,
|
||||
"large commit (refactor or release, not the fix)": 173,
|
||||
"fix spread over several commits or repositories": 167,
|
||||
"merge or root commit": 115,
|
||||
"no source file changed": 42,
|
||||
"language outside the feature vocabulary (go/ruby/c#/...)": 32,
|
||||
"manual review: the chosen function does not hold the fix (comment-only, misattributed, partial or family mismatch)": 19,
|
||||
"repository unavailable": 9,
|
||||
"WordPress plugin (population v3 safety)": 4,
|
||||
"fix commit unavailable": 2
|
||||
},
|
||||
"outcomes_note": "single-family pass (screen 2, every advisory examined, no quota met) plus the two-family pass; `kept` = automated candidates; `manual review: the chosen function does not hold the fix (comment-only, misattributed, partial or family mismatch)` = the screen re-meeting a function the 2026-10-03 review had dropped",
|
||||
"candidates_manually_reviewed": 412,
|
||||
"kept_after_review": 255,
|
||||
"review_drops": {
|
||||
"total": 157,
|
||||
"change in the function unrelated to the advisory (refactor, rename, formatting, another advisory)": 33,
|
||||
"function misattributed (matcher span overrun or an undetected enclosing declaration)": 33,
|
||||
"partial: the function only calls or configures the real fix": 29,
|
||||
"function diff not small (rewrite or refactor around the fix)": 28,
|
||||
"unsure": 17,
|
||||
"family mismatch": 11,
|
||||
"same fix as a kept pair in a fork or sibling repository (would leak across repository folds)": 3,
|
||||
"WordPress plugin (population v3 safety; missed by the automated screen)": 2,
|
||||
"reserved-population guard (flagged by the 2026-10-03 guard run; name not printed)": 1
|
||||
},
|
||||
"one_pair_per_repository_drops": 3,
|
||||
"reserved_population_drops": 2,
|
||||
"kept": 251
|
||||
},
|
||||
"kept": {
|
||||
"total": 251,
|
||||
"by_family": {
|
||||
"access_control": 20,
|
||||
"deserialization": 9,
|
||||
"injection": 56,
|
||||
"output_encoding": 69,
|
||||
"path": 71,
|
||||
"untrusted_destination": 26
|
||||
},
|
||||
"post_cutoff_by_family": {
|
||||
"injection": 5,
|
||||
"output_encoding": 5,
|
||||
"path": 2
|
||||
},
|
||||
"post_cutoff_total": 12,
|
||||
"by_language": {
|
||||
"java": 37,
|
||||
"javascript": 37,
|
||||
"php": 46,
|
||||
"python": 83,
|
||||
"typescript": 48
|
||||
},
|
||||
"post_cutoff_by_language": {
|
||||
"python": 6,
|
||||
"typescript": 6
|
||||
},
|
||||
"by_family_language": {
|
||||
"access_control:java": 5,
|
||||
"access_control:javascript": 3,
|
||||
"access_control:php": 7,
|
||||
"access_control:python": 2,
|
||||
"access_control:typescript": 3,
|
||||
"deserialization:java": 2,
|
||||
"deserialization:php": 1,
|
||||
"deserialization:python": 6,
|
||||
"injection:java": 5,
|
||||
"injection:javascript": 7,
|
||||
"injection:php": 9,
|
||||
"injection:python": 24,
|
||||
"injection:typescript": 11,
|
||||
"output_encoding:java": 7,
|
||||
"output_encoding:javascript": 20,
|
||||
"output_encoding:php": 12,
|
||||
"output_encoding:python": 14,
|
||||
"output_encoding:typescript": 16,
|
||||
"path:java": 10,
|
||||
"path:javascript": 4,
|
||||
"path:php": 10,
|
||||
"path:python": 34,
|
||||
"path:typescript": 13,
|
||||
"untrusted_destination:java": 8,
|
||||
"untrusted_destination:javascript": 3,
|
||||
"untrusted_destination:php": 7,
|
||||
"untrusted_destination:python": 3,
|
||||
"untrusted_destination:typescript": 5
|
||||
},
|
||||
"by_ecosystem": {
|
||||
"composer": 46,
|
||||
"maven": 37,
|
||||
"npm": 81,
|
||||
"pip": 87
|
||||
},
|
||||
"licences": {
|
||||
"Apache-2.0": 80,
|
||||
"BSD-2-Clause": 3,
|
||||
"BSD-3-Clause": 18,
|
||||
"ISC": 1,
|
||||
"MIT": 147,
|
||||
"MPL-2.0": 1,
|
||||
"Unlicense": 1
|
||||
},
|
||||
"split": {
|
||||
"holdout": 124,
|
||||
"train": 127
|
||||
},
|
||||
"published_by_year": {
|
||||
"2023": 50,
|
||||
"2024": 52,
|
||||
"2025": 66,
|
||||
"2026": 83
|
||||
},
|
||||
"published_range": [
|
||||
"2023-01-04",
|
||||
"2026-10-01"
|
||||
],
|
||||
"via_pull_request": 13,
|
||||
"commit_over_120_source_lines": 10,
|
||||
"two_family_advisory_family_chosen_on_review": 3,
|
||||
"verification": {
|
||||
"verified": 251,
|
||||
"excerpt_bytes_read": 1297516
|
||||
},
|
||||
"targets": {
|
||||
"asked": "injection and untrusted_destination >= 60 each; path, access_control, output_encoding, deserialization >= 30 where advisories allow; >= 250 total",
|
||||
"met": {
|
||||
"access_control": false,
|
||||
"deserialization": false,
|
||||
"injection": false,
|
||||
"output_encoding": true,
|
||||
"path": true,
|
||||
"untrusted_destination": false
|
||||
},
|
||||
"limit": "the reviewed-advisory pool, not the review: every advisory since 2023-01-01 in the four ecosystems was examined; the post-cutoff pool was largely spent by batch 2 (its repositories are now excluded), and untrusted_destination/access_control/deserialization advisories mostly lack a linked single fix commit or sit in repositories the benchmarks already use"
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"command": "ousast learn labels --root . --out <scratch> --snapshot <scratch>",
|
||||
"rows": [
|
||||
6981,
|
||||
7985
|
||||
],
|
||||
"pairs_rows": [
|
||||
1936,
|
||||
2940
|
||||
],
|
||||
"new_pair_rows": 1004
|
||||
},
|
||||
"features": {
|
||||
"command": "benchmarks/learn/build_features.py --labels <the 1004 new rows> --units <merged units.json of `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes-3/catalog.toml` (manifests written, no Run started)> --catalog benchmarks/pairs/advisory-fixes-3/catalog.toml --kinds pairs",
|
||||
"records": {
|
||||
"plane:access_control": 40,
|
||||
"plane:deserialization": 18,
|
||||
"plane:injection": 112,
|
||||
"plane:output_encoding": 138,
|
||||
"plane:path": 142,
|
||||
"plane:untrusted_destination": 52,
|
||||
"static:access_control": 40,
|
||||
"static:deserialization": 18,
|
||||
"static:injection": 112,
|
||||
"static:output_encoding": 138,
|
||||
"static:path": 142,
|
||||
"static:untrusted_destination": 52
|
||||
},
|
||||
"instrument_states_plane_all_families": {
|
||||
"agree:missing": 164,
|
||||
"agree:none": 338,
|
||||
"delta:not_applicable": 502,
|
||||
"engine:missing": 406,
|
||||
"engine:none": 96,
|
||||
"entry_points:not_applicable": 502,
|
||||
"facts:not_applicable": 502,
|
||||
"language:ran": 502,
|
||||
"model_sinks:missing": 502,
|
||||
"quick:ran": 502,
|
||||
"roles:ran": 502,
|
||||
"source:ran": 502,
|
||||
"verify:missing": 164,
|
||||
"verify:none": 338
|
||||
},
|
||||
"note": "quick rules and deterministic roles ran on each side; engine, verify, agree and model sinks never ran for these pairs (missing/none: null features, never zeros)"
|
||||
},
|
||||
"memory_build": {
|
||||
"command": "ousast learn memory build --labels <labels> --units <merged units.json> --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)",
|
||||
"examples": [
|
||||
1133,
|
||||
1635
|
||||
],
|
||||
"pairs_examples": [
|
||||
968,
|
||||
1470
|
||||
],
|
||||
"pairs_counts_after": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 1470,
|
||||
"labels": 2940,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 1470,
|
||||
"recorded": 1470,
|
||||
"unread": 0
|
||||
}
|
||||
},
|
||||
"complete_pairs": {
|
||||
"how": "store example rows (profile static) keyed as learn.experiments.freeze_units pairs them: (repo, candidate, family) holding both labels; whole memory, every family",
|
||||
"before": 459,
|
||||
"after": 710,
|
||||
"by_family": {
|
||||
"access_control": [
|
||||
37,
|
||||
57
|
||||
],
|
||||
"config_secrets": [
|
||||
12,
|
||||
12
|
||||
],
|
||||
"deserialization": [
|
||||
32,
|
||||
41
|
||||
],
|
||||
"injection": [
|
||||
68,
|
||||
124
|
||||
],
|
||||
"memory": [
|
||||
149,
|
||||
149
|
||||
],
|
||||
"output_encoding": [
|
||||
77,
|
||||
146
|
||||
],
|
||||
"path": [
|
||||
38,
|
||||
109
|
||||
],
|
||||
"unknown": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"untrusted_destination": [
|
||||
44,
|
||||
70
|
||||
]
|
||||
},
|
||||
"exp_004_005_units": "195 complete pairs inside exp-004/005's frozen 501 units (six families, evaluation folds); those units are frozen and were not regenerated here"
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"before": 99,
|
||||
"after": 99,
|
||||
"groups_both_by_family": {
|
||||
"access_control": [
|
||||
37,
|
||||
57
|
||||
],
|
||||
"config_secrets": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"deserialization": [
|
||||
32,
|
||||
41
|
||||
],
|
||||
"injection": [
|
||||
53,
|
||||
109
|
||||
],
|
||||
"memory": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"output_encoding": [
|
||||
38,
|
||||
107
|
||||
],
|
||||
"path": [
|
||||
38,
|
||||
109
|
||||
],
|
||||
"unknown": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"untrusted_destination": [
|
||||
42,
|
||||
68
|
||||
]
|
||||
}
|
||||
},
|
||||
"embeddings": {
|
||||
"command": "ousast learn memory embed --budget-usd 0.20 (dry run first: 144,806 estimated tokens, $0.0029)",
|
||||
"requested": 1479,
|
||||
"cached": 1004,
|
||||
"embedded": 475,
|
||||
"calls": 15,
|
||||
"tokens": 150339,
|
||||
"usd": 0.003007
|
||||
},
|
||||
"spend": {
|
||||
"chat_model_calls": 0,
|
||||
"embedding_usd": 0.003007,
|
||||
"github_api": "metadata and raw contents only"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user