mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-14 02:08:18 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Marius CiepluchandClaude Opus 5.5
|
e13ce35a75 |
@@ -21,10 +21,3 @@ OPENROUTER_API_KEY=
|
||||
# AWS_SESSION_TOKEN=
|
||||
# The bucket must have versioning enabled, a lifecycle rule expiring runs/, S3 Select and readable object tags
|
||||
# (see docs/rustfs.md); the store verifies them at startup and refuses to run otherwise.
|
||||
|
||||
# Operator notifications (Discord). Status of OpenUltraSAST processes and their results for the
|
||||
# system/AI operator -- not developer debugging, and not k3s/ax infra health. See
|
||||
# .kiro/steering/operator-notifications.md. Webhook posts to the operator channel; the operator id
|
||||
# is the account to @mention on a stuck/impaired alert. Leave blank to disable notifications.
|
||||
DISCORD_WEBHOOK_URL=
|
||||
DISCORD_OPERATOR_ID=
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
name: Engine image
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
paths:
|
||||
- 'plane/Dockerfile.*'
|
||||
- 'plane/task-base.digest'
|
||||
- 'Dockerfile'
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
concurrency:
|
||||
group: engine-image-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
pins:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
base_tag: ${{ steps.pins.outputs.base_tag }}
|
||||
base_digest: ${{ steps.pins.outputs.base_digest }}
|
||||
browser_tag: ${{ steps.pins.outputs.browser_tag }}
|
||||
browser_digest: ${{ steps.pins.outputs.browser_digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
# No login: every pinned base must be anonymously pullable.
|
||||
- id: pins
|
||||
run: |
|
||||
python3 - <<'PYTHON'
|
||||
import json, os, re, subprocess
|
||||
from pathlib import Path
|
||||
pins = json.loads(Path("plane/task-base.digest").read_text())
|
||||
manifests = {}
|
||||
for key, output in (("base", "base"), ("base-browser", "browser")):
|
||||
ref = pins.get(key)
|
||||
pattern = r"ghcr.io/norandom/ousast-task-base:(base-[a-z0-9-]+)@(sha256:[0-9a-f]{64})"
|
||||
match = re.fullmatch(pattern, ref or "")
|
||||
if not match:
|
||||
raise SystemExit(f"{key}: unpublished/invalid pin; run task-base-image and update plane/task-base.digest")
|
||||
raw = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", "--raw", ref])
|
||||
manifests[key] = json.loads(raw)["layers"]
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
|
||||
out.write(f"{output}_tag={match[1]}\n{output}_digest={match[2]}\n")
|
||||
base = manifests["base"]
|
||||
if not base or manifests["base-browser"][:len(base)] != base:
|
||||
raise SystemExit("Browser pin does not share the pinned base's layers")
|
||||
PYTHON
|
||||
image:
|
||||
needs: pins
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- package: openultrasast
|
||||
dockerfile: Dockerfile
|
||||
- package: ousast-search-task
|
||||
dockerfile: plane/Dockerfile.search-task
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
PACKAGE: ${{ matrix.package }}
|
||||
run: |
|
||||
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
- id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=${{ matrix.package }}
|
||||
type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.package }}
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
- env:
|
||||
IMAGE: ${{ steps.names.outputs.image }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
engine:
|
||||
needs: pins
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
image: ${{ steps.names.outputs.image }}
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
PACKAGE: ousast-engine-task
|
||||
run: |
|
||||
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
- id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.engine-task
|
||||
target: engine
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=ousast-engine-task
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
- env:
|
||||
IMAGE: ${{ steps.names.outputs.image }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
replay:
|
||||
needs: [pins, engine]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
PACKAGE: ousast-replay-task
|
||||
run: |
|
||||
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
- id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.engine-task
|
||||
target: replay
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=ousast-replay-task
|
||||
type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=ousast-replay-task
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
- env:
|
||||
IMAGE: ${{ steps.names.outputs.image }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
@@ -1,78 +0,0 @@
|
||||
name: Task base image
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'plane/Dockerfile.base'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
concurrency:
|
||||
group: task-base-image
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
base:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
run: |
|
||||
echo "tag=base-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
content=$(cat plane/Dockerfile.base pyproject.toml uv.lock | sha256sum | cut -d' ' -f1)
|
||||
echo "content=base-content-$content" >> "$GITHUB_OUTPUT"
|
||||
- id: base
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.base
|
||||
target: base
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
sbom: false
|
||||
tags: |
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}
|
||||
cache-from: type=gha,scope=task-base
|
||||
cache-to: type=gha,mode=max,scope=task-base
|
||||
- id: browser
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.base
|
||||
target: base-browser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
TASK_BASE_REF=ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||
tags: |
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}-browser
|
||||
cache-from: type=gha,scope=task-base-browser
|
||||
cache-to: type=gha,mode=max,scope=task-base-browser
|
||||
- name: Print manual pin update and verify public access
|
||||
env:
|
||||
BASE: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||
BROWSER: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser@${{ steps.browser.outputs.digest }}
|
||||
run: |
|
||||
python3 - <<'PYTHON' >> "$GITHUB_STEP_SUMMARY"
|
||||
import json, os
|
||||
print("Replace plane/task-base.digest with this JSON after both public checks pass:")
|
||||
print("```json")
|
||||
print(json.dumps({"base": os.environ["BASE"], "base-browser": os.environ["BROWSER"]}, indent=2))
|
||||
print("```")
|
||||
PYTHON
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$BASE"
|
||||
docker buildx imagetools inspect "$BROWSER"
|
||||
@@ -36,9 +36,3 @@ workspace/
|
||||
|
||||
# MkDocs build output (mkdocs build)
|
||||
site/
|
||||
|
||||
# Unseen evaluation pointers for non-permissive repositories.
|
||||
benchmarks/unseen/private/
|
||||
|
||||
# Feasibility pilot identities remain local; only aggregate records are tracked.
|
||||
benchmarks/search/private/
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
# Engine change-scoped analysis (G2 workstream)
|
||||
|
||||
Status: **proposed / in progress** (opened 2026-10-11). Owner: engine. Tracks a concrete lever for the
|
||||
engine-detection gap found while driving the G0 baseline. This is G2 ("improve what moves the unseen-repo
|
||||
numbers"), gated on the pool metric; it is NOT part of the G0 record-07 baseline (that runs on the quick-rule tier).
|
||||
|
||||
## Problem (measured, not assumed)
|
||||
|
||||
On real unseen vulnerability changes the engine produces **no finding** — but the diagnosis (live, 300 s long-deadline
|
||||
replays) shows it is usually **not a detection-capability gap**:
|
||||
|
||||
- **Clean Python injection change** (`aws/bedrock-agentcore-sdk-python`, `install_packages`, introducing, 3-file
|
||||
diff): `wall=303 s`, hit the deadline. The ranker found candidates (`static_rank=1.0`) but **`selected: []`** and
|
||||
every candidate `deferred` with `reason: deadline_exhausted`, `analysis_not_started=deadline_exhausted`,
|
||||
`change_context_incomplete`. **Zero analyses ran.** And the ranked candidates were **repo-wide and unrelated to
|
||||
the push** (`auth.py` access_control/config_secrets), not the changed `install_packages`. The engine is trying to
|
||||
re-audit the whole repo and runs out of budget before analyzing the change.
|
||||
- **Noisy last_touch config_secrets change** (`jenkinsci/metrics-plugin`, 14-file diff): finished in 206 s but
|
||||
`comparison_unknown` + `capability_unavailable` + `eligibility_provenance_mismatch` +
|
||||
`contributor-scan ... projection_unavailable`. Multi-causal; correspondence was one part (fixed separately,
|
||||
commit c39fbb8), capability registry is Task 8.3.
|
||||
|
||||
## Hypothesis
|
||||
|
||||
A pre-push hook should vet **the code this push changed**, not re-scan the whole repository. Seeding/scoping the
|
||||
engine's analysis to the **changed declarations and their dataflow neighborhood** (the `change_sites` are already in
|
||||
the replay spec) should let analysis **actually run within budget** instead of being deferred — which is where
|
||||
detection is being lost, not in Joern's capability.
|
||||
|
||||
## Anchors (confirm in increment 1)
|
||||
|
||||
- `src/openultrasast/rank.py` — the static-evidence ranker (`ranking_mode="evidence"`, `existing-static-evidence-v1`).
|
||||
- `src/openultrasast/mapping.py` — builds the `selected`/`deferred` scope + the budget allocation that defers on
|
||||
`deadline_exhausted`.
|
||||
- `src/openultrasast/push/policy.py`, `push/runner.py` — consume the scope; change_sites arrive via the push input.
|
||||
- `benchmarks/unseen/replay_entry.py` — carries `changed_paths` / `declaration_paths` in the spec.
|
||||
|
||||
## Increments (progress tracking)
|
||||
|
||||
- [x] **0. Cost split confirmed (2026-10-11, timed live replay).** `build_seconds=66.69` (CPG build is cheap),
|
||||
`query_seconds=0.0` (no query ever executed), total 303 s (hit the deadline), **4,095** candidate questions all
|
||||
`analysis_not_started=deadline_exhausted`, `selected: []`, `regions_scanned=0`. So of the 303 s, ~67 s built the
|
||||
CPG and ~236 s went to enumerating/ranking/preparing 4,095 repo-wide candidate questions for a **3-file** change —
|
||||
the budget was spent before a single query ran. The CPG build is NOT the bottleneck; the over-broad candidate set
|
||||
is. Change-scoping is confirmed as the right lever, and because per-candidate prep includes the line-correspondence
|
||||
work, fewer candidates also cuts that cost (the two fixes compound).
|
||||
- [x] **1. Located (2026-10-11).** The over-enumeration is upstream of the scan, not inside it:
|
||||
- `push/runner.py:345-348` — `head_regions = discovery(tip.root, …)` discovers regions across the **whole head
|
||||
tree** (the ~4,095), independent of the change.
|
||||
- `push/runner.py:397-407` — those regions are passed wholesale to `scan_repository(..., change_context=context)`;
|
||||
`change_context` is threaded in but **never scopes the region set**.
|
||||
- `model/scan.py:167` and `:364` — `[_identity(unit, r, f) for r in regions for f in r.families]` enumerates all
|
||||
regions × families; with `order_by_evidence=True` (`runner.py:288/403`) the scan evidence-tiers **every** pair
|
||||
before analysis — that pre-work is the ~236 s measured in increment 0, so the deadline hits at `query_seconds=0`.
|
||||
- Primitive already present: `model/regions.py:279` computes the changed-path set from the context;
|
||||
`compare_targeted_base` (`runner.py:411`) is a precedent for change-targeted work.
|
||||
- **Fix locus (increment 2):** restrict the *analyzed* region set to the change's declarations + dataflow
|
||||
neighbourhood (filter `head_regions` by `context.changed_paths` before the scan, or scope the enumeration inside
|
||||
`scan_repository`). Keep the **CPG whole-repo** so taint queries can still trace sources from unchanged code into
|
||||
the changed regions — only the question set is change-scoped.
|
||||
|
||||
**Correction (2026-10-11), after a reverted first attempt — two things the naive filter got wrong:**
|
||||
- **The engine already reaches into unchanged code by design, and it is tested.** A changed `source` that reaches
|
||||
an unchanged `sink` must still select the unchanged sink's question
|
||||
(`tests/test_model_change_context.py::test_changed_dependency_reaches_unchanged_ranked_question`;
|
||||
`test_change_does_not_replace_ranker_selection` pins that change context must NOT alter the ranker's selection).
|
||||
`_changed_files` feeds `_evidence_pass` only as `contextFilter`/`contextPaths` — which locations to itemise vs
|
||||
summarise — NOT to shrink the tier population. A "changed files only" scope deletes the reach-into-unchanged
|
||||
behaviour. Any scope must keep changed regions **plus the regions they reach** (the affected/`relationships`
|
||||
neighbourhood), not changed files alone.
|
||||
- **The 236 s is now split (2026-10-11, saved artifacts).** `total=291s build=52s query=0s arbitrate=238.75s`,
|
||||
and `query_seconds_by_kind["evidence"]=238.54s` — the evidence-tier pass is **99.9 %** of the arbitrate phase;
|
||||
`affected_context` + `_scope_work` are **0.21 s**. The pass tiered 1,015 pairs (`tier_counts 0:1009, 1:6`) at
|
||||
**~235 ms/pair** (contended; ~120 ms uncontended) and that alone blew the budget — `selected=0`,
|
||||
`deferred=1365` all `deadline_exhausted`. Secondary anomaly: the design assumes ~2.5 ms/pair
|
||||
("17,700 in 44 s"); here it is 50–100× that, i.e. the evidence query is pathologically slow on a cold large
|
||||
CPG — a separate optimisation from cutting the pair count.
|
||||
- **So increment 2's single target is the evidence pass** (`scan.py:338-359`, `_evidence_pass`). The fork is HOW
|
||||
to cut it without deleting reach-into-unchanged:
|
||||
- (i) **Scope + bound:** order the evidence candidates change-first and bound the pass to a fraction of the
|
||||
budget so analysis queries still run; reached unchanged regions are tiered only while budget lasts.
|
||||
- (ii) **Neighbourhood pass:** tier the changed regions, read the reached regions from the context rows the
|
||||
pass already returns, tier those in a second small pass — exact reach-into-unchanged, more code.
|
||||
- (iii) **Per-pair speedup:** attack the ~235 ms/pair (warm server / batching) rather than the pair count —
|
||||
helps every scan but is a deeper engine change and does not by itself bound a huge repo.
|
||||
|
||||
**CORRECTION (2026-10-11, after implementing (i) + four live re-verifies — the cost model was wrong).**
|
||||
Scope + bound was built and tested (change-first ordering, adaptive budget-fitted portioning, `_context_pass`
|
||||
scoped to changed files, honest `evidence_pass_bounded` degradation; branch `engine-change-scoped-evidence-pass`).
|
||||
It correctly cut the tiered pair count — but `query_seconds` stayed **0** every time. The live numbers show why:
|
||||
|
||||
| pairs tiered | run_batch calls | evidence s | build s |
|
||||
|---|---|---|---|
|
||||
| 1015 | 1 | 238 | 52 |
|
||||
| 512 | 1 | 219 | 68 |
|
||||
| 64 | 2 | 220 | 68 |
|
||||
| 4 (probe) | 1 | 160 | 127 |
|
||||
|
||||
Evidence time is **flat regardless of pair count** → the cost is **per-*call* fixed** (~150–220 s: loading the CPG
|
||||
into the query engine + the census the evidenceOnly batch computes), ~85 ms/pair on top. Pair-count scoping
|
||||
therefore cannot free budget, and portioning re-pays the fixed cost per call (so (i) can be counterproductive).
|
||||
Build is also large and swings 52→127 s — two engines thrashing the 7 GB host (the record-07 baseline shares it).
|
||||
The real levers are now (iii)-shaped, not (i): **warm/persistent query engine or a cached census** to kill the
|
||||
~150 s per-call fixed cost, and/or a **bigger engine-arm budget** than 300 s; plus a **clean (uncontended)
|
||||
measurement**, because contention alone may be inflating build + fixed cost 2–3×. Scope+bound is NOT merged.
|
||||
|
||||
**UNCONTENDED measurement (2026-10-11, baseline paused via SIGSTOP + container kill, then resumed).** Shipped
|
||||
engine, one evidence call over 1,015 pairs: `build=48 s`, `evidence=243.79 s`, `query=0`. The evidence pass is
|
||||
**the same ~244 s uncontended as ~238 s contended** → the cost is **INTRINSIC, not contention**; contention only
|
||||
inflated `build` (48 → 127 s) and added run-to-run noise. The model is `evidence ≈ 150 s fixed (CPG load into the
|
||||
query engine + the census the evidenceOnly batch computes) + ~93 ms/pair` (244 = 150 + 1015×0.093). Consequences:
|
||||
- **~200 s of fixed overhead** (`build` + evidence-load) precedes any analysis, so a 300 s budget leaves only
|
||||
~100 s for everything else. This is why `query=0` even on a clean host.
|
||||
- **Change-scoping still helps, but only by cutting the per-pair tail** (~1,015×93 ms ≈ 95 s): one evidence call
|
||||
over ~30 changed pairs is ~150 s + 3 s instead of 244 s, which would free ~90 s — possibly enough for analysis,
|
||||
but marginal.
|
||||
- **It must be ONE call, not portioned.** The ~150 s is a first-call load, so the adaptive probe attributed it to
|
||||
its 4 probe pairs (per-pair≈37 s), concluded the budget fit ~0 more, and tiered only 4 — the implemented
|
||||
scope+bound is doubly wrong for this cost model. The correct shape: change-first candidates, count-capped, ONE
|
||||
evidence call, `deadline=None`.
|
||||
- **Pre-push viability:** the ~200 s fixed floor means the engine arm cannot be a fast (≤30 s) pre-push check on a
|
||||
repo this size without a warm/persistent session or a cached census. A larger budget (e.g. 600 s) makes the
|
||||
benchmark arm run but is not a realistic pre-push deadline. The lever that matters for the product is killing
|
||||
the ~150 s fixed cost, not scoping pairs.
|
||||
- [ ] **2. Seed analysis from the change.** Prioritize (or restrict first-pass to) the changed declarations plus
|
||||
their call/data neighborhood over repo-wide static rank, within budget.
|
||||
- [ ] **3. Soundness.** Keep cross-file dataflow INTO the changed function (don't scope to literal changed lines);
|
||||
add tests; keep existing push tests green.
|
||||
- [ ] **4. Live re-test.** Re-run the bedrock change: does analysis now RUN and produce a finding on
|
||||
`install_packages` (vs `deferred`)? Re-run metrics-plugin.
|
||||
- [ ] **5. Measure the delta.** Engine-arm sample on the pool (catch/false-alarm + engine-coverage rate) — needs the
|
||||
kube lane (blocked on the AX completion path) for scale.
|
||||
|
||||
## Dependencies / notes
|
||||
|
||||
- Coverage `complete_within_scope` also needs the capability registry (Task 8.3) regardless of this work; the win
|
||||
here is findings moving from `comparison_unknown`/deferred to a decided novelty → advisory engine finding.
|
||||
- Implement directly and verify on the LIVE instrument (a fake-client/unit pass is not evidence the live engine
|
||||
changed); the kube-lane and store endpoints are operator-owned infra, not assumptions to make here.
|
||||
</content>
|
||||
@@ -1,80 +0,0 @@
|
||||
# Plan: learn fix mechanisms into the engine (draft for review, 2026-10-04)
|
||||
|
||||
Status: approved 2026-10-04 by the maintainer. Increment 1 = discharge facts only; coverage additions (Req 7) come later.
|
||||
Decisions: minimum 2 proposing repositories before a candidate is tried; admission per language (not pooled by callee name).
|
||||
|
||||
## Why
|
||||
|
||||
The Joern trace run over exp-005's 501 units shows the engine cannot tell a vulnerable function from its
|
||||
fix. In the 105 pairs where both sides were analysed, a taint path appears on both sides in 12, on the
|
||||
vulnerable side only in 2, on the fixed side only in 0, and on neither in 91
|
||||
(`benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json`).
|
||||
|
||||
So the fix is invisible to the engine. Either the flow survives and the engine does not know the fix's
|
||||
mechanism (an `allowed_classes` argument, a path normalisation check, an escaping call), or the engine's
|
||||
sources and sinks miss the function on both sides. A classifier fed this evidence cannot improve on it.
|
||||
|
||||
The corpus now holds 710 complete fix pairs (`2026-10-04-decision-engine-harvest-3/record.json`). Each fix
|
||||
diff shows what was added. That is the material the engine lacks.
|
||||
|
||||
## Goal
|
||||
|
||||
Grow the engine's discharge knowledge (sanitizers, guards, safe-argument forms) and, second, its source and
|
||||
sink coverage, from the fix diffs. Admit a fact only when it improves held-out separation. Spend no model
|
||||
money.
|
||||
|
||||
## Requirements (proposed)
|
||||
|
||||
1. **Mining.** From each fix pair, extract what the fixed function adds on the vulnerable flow: new calls,
|
||||
new conditions guarding the sink, new or changed sink arguments. Record each as a candidate fact with its
|
||||
language, family, form (sanitizer call, guard predicate, safe argument, sink restriction) and the pairs
|
||||
that propose it. Mining is deterministic (tree-sitter diff of the two function bodies). No model calls.
|
||||
2. **Generalisation is explicit.** A candidate is a pattern, not a repository name: a callee name, an
|
||||
argument key, a predicate shape. Candidates naming project-specific identifiers are rejected (Req 8: no
|
||||
application-specific trees).
|
||||
3. **Admission by held-out separation.** Folds are by repository, as in the decision engine. A candidate is
|
||||
admitted for a fold only if, on that fold's held-out pairs, adding it to the fact tables:
|
||||
- increases pairs where the vulnerable side keeps a path and the fixed side loses it or becomes
|
||||
sanitized;
|
||||
- and does not remove any held-out vulnerable-side path (no recall loss).
|
||||
A candidate proposed by fewer than 2 distinct repositories in the training folds is not tried.
|
||||
4. **No train-on-test.** Mining and admission never read held-out pairs; the closed-loop leak recorded in
|
||||
memory (`closed-loop-train-on-test-leak`) is the failure this rule exists to prevent. Population v3 is
|
||||
never read.
|
||||
5. **Measurement by the trace runner.** Each admission round re-runs the engine on the affected pins only
|
||||
(the runner's resume and selection already support this; on k3s the worker Deployment runs them in
|
||||
parallel). The outcome is the pair-asymmetry table per family, before and after, on held-out folds.
|
||||
6. **Gate before production.** An admitted fact enters `src/openultrasast/ruleset/semantic/*.toml` only after
|
||||
the detection gates (`gate`, `map_gate`, `pair_gate`) show no regression, and only as a registered
|
||||
experiment whose decision rule was fixed before the run.
|
||||
7. **Coverage second.** After discharge facts, the same loop proposes source and sink additions from pairs
|
||||
where neither side has a path (91 of 105 today), admitted under the same held-out rule.
|
||||
|
||||
## How it relates to what exists
|
||||
|
||||
- `corpus-seeded-mechanisms` already has a closed loop that admits and retracts mechanisms by leave-one-out.
|
||||
This plan reuses its lever and store, and points it at discharge facts mined from real fix diffs instead
|
||||
of variant search.
|
||||
- `benchmarks/pairs/mechanisms.toml` (23 mechanisms) labels what kind of bug a pair is. The new candidates
|
||||
describe what a fix does. They live in the fact tables, not in the mechanism vocabulary.
|
||||
|
||||
## Measures of success
|
||||
|
||||
- Pair asymmetry on held-out folds: vulnerable-only paths rise from 2 of 105 toward a substantial share,
|
||||
with zero fixed-only paths.
|
||||
- No held-out recall loss on vulnerable sides.
|
||||
- Then the decision engine is re-run with the new engine evidence (exp-005 design), and the pooled block gate
|
||||
(exp-004 design) is re-measured.
|
||||
|
||||
## Cost and time
|
||||
|
||||
- Model cost: $0 (mining and admission are deterministic; the classifier re-run afterwards is a separate,
|
||||
approved spend).
|
||||
- Engine time: one pass over affected pins per admission round; about 2 hours per full pass on k3s plus the
|
||||
VM, about 6 hours on the VM alone.
|
||||
|
||||
## Open questions for the maintainer
|
||||
|
||||
1. Should coverage additions (Req 7) be in the same increment or a later one?
|
||||
2. Minimum proposing repositories: 2 is the proposal. Higher is safer, lower finds more.
|
||||
3. Admission per family and language, or pooled across languages for the same callee name?
|
||||
@@ -132,13 +132,6 @@ rule hits.
|
||||
- **t_quiet**: per family, the highest threshold keeping >= 90% of the paired positives at or above it (chosen
|
||||
on training folds, its held-out recall reported); findings below it are silent.
|
||||
- No held-out label moves a threshold or a normalisation map; candidates whose verdict is `unsure` never block.
|
||||
5. *(Amended 2026-10-05, maintainer decision, with `search-with-proof`.)* Where a family has an owned demonstration
|
||||
oracle, the engine is **triage, not the gate**: BLOCK comes only from a verified demonstration (`search-with-proof`
|
||||
Req 3, 6.3), and the engine's job is to put the true vulnerable function among the top K candidates of a push.
|
||||
For those families the engine is judged by (a) top-K recall per push (K = 3 to start) on the unseen-repo pool and
|
||||
(b) its advisory false-alarm rate there; the pooled block gate of 6.4 then governs only families without an
|
||||
oracle (access control, deserialisation, configuration and secrets), where the engine remains the only signal.
|
||||
Every arm reports top-K recall next to its catch and false-alarm rates.
|
||||
|
||||
### Requirement 7: Built for unseen repositories -- a repo safety net
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"feature_name": "learned-decision-engine",
|
||||
"created_at": "2026-09-30T14:00:00Z",
|
||||
"updated_at": "2026-10-04T18:00:00Z",
|
||||
"updated_at": "2026-10-01T10:20:00Z",
|
||||
"language": "en",
|
||||
"phase": "implementation",
|
||||
"approvals": {
|
||||
|
||||
@@ -8,10 +8,6 @@ memory, compiled DSPy-style in-house); tasks 1-5 are unaffected, tasks 6-10 are
|
||||
against the provider balance read just before it, and none starts with a ceiling above that balance. No test makes
|
||||
a model or embedding call.
|
||||
|
||||
Direction 2026-10-04: `.kiro/steering/roadmap.md` (section of 2026-10-04, phases G0-G5) is the authority. Every
|
||||
change is judged on the unseen-repository pool (G0, spec `unseen-repo-evaluation`) by catch rate and false-alarm
|
||||
rate per push. Within-pair AUC stays a development signal. Tasks 11-14 below follow that roadmap.
|
||||
|
||||
- [x] 1. Schema and features
|
||||
- `learn/schema.py` (closed, typed allow-list; no identities; missing != zero), `learn/features.py` (host
|
||||
builder), new memory `KINDS`, `plane/tasks/features.py` and the `remember` input; tests enforcing the
|
||||
@@ -49,11 +45,6 @@ rate per push. Within-pair AUC stays a development signal. Tasks 11-14 below fol
|
||||
- _Requirements: 1.1_
|
||||
|
||||
- [ ] 6. Program, local memory and compilation (design sections 4-5)
|
||||
- Status 2026-10-04: 6.1-6.6 are done. 6.7 is partial: leave-one-source-out, the 0/25/50% memory-size points and
|
||||
the source-count curve are not run. Around this task the experiment machinery grew: rule arms
|
||||
(`precision_bound_block`, `pooled_block_gate` with a family floor) in `learn/experiments.py`, the response-cache
|
||||
fill with a hard ceiling and a dry-run estimate in `learn/fill.py`, and the in-function slice in `learn/slice.py`.
|
||||
These ran exp-003, exp-004 and exp-005 (7.5-7.7). Memory now holds 710 complete fix pairs (7.9).
|
||||
- [x] 6.1 Excerpts and the example store (no model)
|
||||
- `learn/excerpt.py` (function span at the label's pin, bounds 80/4,000 for candidates and 40/2,000 for
|
||||
examples, numbered lines, no path/repository/commit, advisory ids and security wording in comments
|
||||
@@ -160,28 +151,6 @@ rate per push. Within-pair AUC stays a development signal. Tasks 11-14 below fol
|
||||
rows' null plane-only features (1f525ff), the 788 embedding vectors absent from RustFS (copied from the file
|
||||
store), the result row's column-type collision under S3 Select (780caaa). Further program variants (`k = 5` vs
|
||||
`k = 1`, the confidence source) as new manifests.
|
||||
- [x] 7.5 exp-003 run and recorded 2026-10-03 ($0, replay only): **inconclusive**. Selective blocking (a per-family
|
||||
raw-score threshold chosen on training folds by its Wilson lower bound) offers no BLOCK on held-out paired units
|
||||
in any family; the incumbent stays. 211 of 353 frozen units evaluated (142 replay misses). Proving 0.95 needs at
|
||||
least 73 clean flags. Result: `benchmarks/experiments/exp-003-precision-bound-blocking/result.json`.
|
||||
- [x] 7.6 exp-004 run and recorded 2026-10-03: **inconclusive**. One pooled BLOCK threshold over all families with a
|
||||
family floor (Req 6.4). No outer fold found a threshold, pooled or per family; B offers no BLOCK. 501 units,
|
||||
replay at $0 after a cache fill of $1.24 (ceiling $2; the dry-run estimate over-priced 4.5x). Result and record:
|
||||
`benchmarks/experiments/exp-004-pooled-block-gate/result.json`, `record.json`.
|
||||
- [x] 7.7 exp-005 run and recorded 2026-10-03: **inconclusive**. The in-function def-use slice in the prompt of the
|
||||
six slice-2 programs. Within-pair AUC 0.762 -> 0.746 (CI [-0.063, 0.032], spans 0); unsure rate 0.29 -> 0.24
|
||||
(favours B); usd per candidate slightly higher. A $0 leak audit came first
|
||||
(`benchmarks/measurements/2026-10-03-graph-slice-leak-audit/record.json`). Result:
|
||||
`benchmarks/experiments/exp-005-graph-slice/result.json`.
|
||||
- [x] 7.8 Negatives harvest 2026-10-03 ($0.0014 embeddings, no chat-model call): 98 real fix pairs whose function
|
||||
survives the fix, from reviewed advisories, as pointer-pair corpus `advisory-fixes-2`; memory 937 -> 1133
|
||||
examples. Record: `benchmarks/measurements/2026-10-03-decision-engine-negatives-harvest/record.json`.
|
||||
- [x] 7.9 Harvest 3, 2026-10-04 ($0.003 embeddings, no chat-model call): 251 pairs as `advisory-fixes-3`; complete
|
||||
pairs in memory 459 -> 710. Record: `benchmarks/measurements/2026-10-04-decision-engine-harvest-3/record.json`.
|
||||
- [x] 7.10 Joern trace run over exp-005's 501 units and its leak audit, 2026-10-04 ($0): no leak flagged. In the 105
|
||||
pairs where both sides were asked, a path appears on both sides in 12, on the vulnerable side only in 2, on the
|
||||
fixed side only in 0, and on neither in 91. The engine cannot see what a fix changed. Record:
|
||||
`benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json`.
|
||||
- _Requirements: 3.6, 4.1-4.4_
|
||||
|
||||
- [ ] 8. Integration
|
||||
@@ -207,60 +176,3 @@ rate per push. Within-pair AUC stays a development signal. Tasks 11-14 below fol
|
||||
`result-v3-engine.json` is committed. Nothing between the two commits may touch the program, memory snapshot,
|
||||
features, priors or thresholds. This spec's code never opens v3 files.
|
||||
- _Requirements: 5.3_
|
||||
|
||||
- [ ] 11. Fix mechanisms (plan approved 2026-10-04; see `fix-mechanisms-plan.md`)
|
||||
- Roadmap G2 arm (`.kiro/steering/roadmap.md`, section of 2026-10-04): one candidate improvement, not the spine.
|
||||
It is judged on the unseen-repository pool (G0, spec `unseen-repo-evaluation`) by catch rate and false-alarm
|
||||
rate per push. **Paused after 11.1 until the G0 pool exists.**
|
||||
- [ ] 11.1 Mine candidate discharge facts from the 710 fix diffs (sanitizer calls, guarding conditions, safe sink
|
||||
arguments, safe replacements); deterministic, $0. In progress.
|
||||
- [ ] 11.2 Admission by held-out separation per outer fold: a candidate turns "path on both sides" into "path on the
|
||||
vulnerable side only" on folds it was not mined from, removes no held-out vulnerable path, has at least 2
|
||||
proposing repositories, and is admitted per language.
|
||||
- [ ] 11.3 Engine re-run on the affected pins (kube-ax lanes plus the VM lane).
|
||||
- [ ] 11.4 Registered experiment, with the detection gates green, before `ruleset/semantic/*.toml` changes.
|
||||
- [ ] 11.5 Coverage increment: source and sink additions for pairs where neither side has a path (later).
|
||||
- [ ] 11.6 TypeScript admission in the engine (branch `engine-typescript`).
|
||||
- _Requirements: 4.1-4.4, 7.1, 8.1_
|
||||
|
||||
- [ ] 12. exp-006: Joern witness slice in the prompt **(budget go-ahead, about $2-4)**
|
||||
- Planned, roadmap G1/G2. Against the current prompt, after a $0 leak audit, on re-frozen units. Judged on the
|
||||
unseen-repository pool, not on within-pair AUC alone.
|
||||
- _Requirements: 4.1-4.4, 7.1_
|
||||
|
||||
- [ ] 13. exp-007: the combiner ($0 replay)
|
||||
- Planned, roadmap G1/G2. Three or four weights (score, engine entailment, neighbour agreement) over cached
|
||||
responses. Judged on the unseen-repository pool.
|
||||
- _Requirements: 4.1-4.4, 7.1_
|
||||
|
||||
- [ ] 14. Re-run the pooled block gate on re-frozen units
|
||||
- Planned, roadmap G3. The exp-004 design with the family floor (Req 6.4), together with the pool's false-alarm
|
||||
rate per family.
|
||||
- _Requirements: 6.4, 7.3_
|
||||
|
||||
- [ ] 15. Trained classifier arm (maintainer decision 2026-10-04: train a model, not hand-set profiles; judged on the
|
||||
unseen-repo pool, roadmap G1)
|
||||
- [ ] 15.1 Features: code embedding of the excerpt (stored vectors) + every allow-listed signal; no hand-picked
|
||||
profile. Automated screening: a signal that separates pair sides by itself on training folds above the leak
|
||||
threshold is dropped for that fold (the leak audit as a filter, fitted per fold).
|
||||
- [ ] 15.2 Models: regularised logistic regression and gradient-boosted trees, fitted per repository-held-out outer
|
||||
fold with inner-fold hyper-parameter search; per-family and pooled variants.
|
||||
- [ ] 15.3 Scores on held-out folds: within-pair AUC, pooled AUC, top-k precision, Wilson bound at the pooled
|
||||
threshold; compared with the current LLM programs on the same units.
|
||||
- [ ] 15.4 Scored on the unseen-repo pool when G0 exists: catch rate at fixed false-alarm budgets.
|
||||
- _Requirements: 3 (revised 2026-10-04 by maintainer decision), 6, 7_
|
||||
|
||||
- [ ] 16. Jointly optimised LLM program arm (judged on the unseen-repo pool)
|
||||
- Search input fields, retrieval parameters, k, instruction and demonstrations together (DSPy MIPRO-style) on
|
||||
training folds; no hand-set input profile. Budget go-ahead before paid search.
|
||||
- _Requirements: 3, 6, 7_
|
||||
|
||||
- [ ] 17. Triage objective (Req 6.5, amended 2026-10-05 with `search-with-proof`)
|
||||
- [ ] 17.1 Metric: top-K recall per push (K = 1, 3, 5) of the true vulnerable function among changed functions,
|
||||
on held-out folds and on the unseen-repo pool; added to the pool scorer next to catch and false alarm.
|
||||
- [ ] 17.2 Score the existing arms on it ($0 replay): trained logistic (cheapest per candidate, likely default),
|
||||
LLM programs (exp-005 A), engine-evidence ordering, and an averaged ensemble.
|
||||
- [ ] 17.3 Re-scope 14 (pooled block gate) to families without a demonstration oracle; 15.4 and 16 report top-K
|
||||
recall as their primary unseen-pool number for families with an oracle.
|
||||
- _Requirements: 6.5, 7_
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
# Brief: MCP shift-left agent + skills
|
||||
|
||||
Status: brief / not yet specced. Proposed 2026-10-07. Delivery surface for the
|
||||
qualified safety net; builds on the existing `ousast mcp` server and the product
|
||||
skills. Related: `proof-arm-opencode` (the proof model), `pre-push-safety-net`
|
||||
(the gate), `unseen-repo-evaluation` (the qualification).
|
||||
|
||||
## Motivation
|
||||
|
||||
The pre-push hook is the gate. Developers and their coding agents want the same
|
||||
security feedback **earlier**, while the code is still being written, not only at
|
||||
push. And in an agent-driven workflow, a mandatory security check is how a human
|
||||
oversees agents: the agent's output has to clear it before it ships.
|
||||
|
||||
Proposal: expose the safety net through an MCP agent and a small set of skills so
|
||||
an editor or a coding agent can ask for a security read of the changed code on
|
||||
demand, shift-left, and so a coding agent can be required to pass it before it
|
||||
proposes a push.
|
||||
|
||||
## Shape
|
||||
|
||||
- **Transport:** extend the existing narrow MCP server (`ousast mcp`, stdio) that
|
||||
OpenCode and other agents already speak. No new network surface; same
|
||||
no-secret, bounded contract.
|
||||
- **What the agent offers:**
|
||||
1. Scan the current diff with the static arm (fast, no model, no cluster) and
|
||||
return findings scoped to the change, with location and a concrete repair
|
||||
direction, not a confidence label.
|
||||
2. Triage a finding: explain it in plain terms and whether it is reachable.
|
||||
3. Prove on demand: hand the top candidate to the proof arm when a runnable
|
||||
harness exists, and return the demonstrated proof-of-concept.
|
||||
- **Skills:** the existing `openultrasast-scan` and `openultrasast-triage`
|
||||
product skills, plus shift-left additions (scan-the-diff, explain-a-finding,
|
||||
suggest-a-fix-direction) that a coding agent invokes mid-task.
|
||||
- **Oversight mode:** a coding agent can be configured to call the scan as a
|
||||
required step before it finalizes a change, so security review is automatic
|
||||
rather than remembered.
|
||||
|
||||
## Control (same model as the hook)
|
||||
|
||||
Advisory by default; blocking is opt-in. Budget and a push/call deadline bound
|
||||
the work. The inline path runs the fast static arm only; heavy proof is explicit
|
||||
and lane-selected. Models and spend caps stay in `.env`.
|
||||
|
||||
## Open questions for the spec
|
||||
|
||||
1. Inline latency budget: in-editor feedback must return in seconds, so the
|
||||
static arm must be fast on a single diff. What is the time contract, and does
|
||||
the large-repo latency problem block it?
|
||||
2. Return contract: exactly what a coding agent gets back (findings, locations,
|
||||
repair direction, optional proof) and how it decides pass/fail in oversight
|
||||
mode. The return should be ticket-ready evidence a busy developer can paste
|
||||
into the issue tracker to win capacity in a prioritization discussion with a
|
||||
product owner, since a reproducible proof is what moves a security fix out of
|
||||
the backlog.
|
||||
3. Does oversight mode need a signed/attestable result so a reviewer can trust
|
||||
that the agent actually ran the check?
|
||||
4. Reuse: the `ousast mcp` tool set already exists; which tools are enough and
|
||||
what is genuinely new.
|
||||
|
||||
## Scope and sequencing
|
||||
|
||||
This is a delivery and adoption surface, correctly **after** the safety net is
|
||||
qualified on unseen repositories (G0 through G5). It is not a detection change;
|
||||
it is how developers and agents consume the detection early. The proof model
|
||||
improvement lives in `proof-arm-opencode`; this brief is the dev- and
|
||||
agent-facing feedback channel.
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"feature_name": "mcp-shift-left-agent",
|
||||
"created_at": "2026-10-07T15:10:00Z",
|
||||
"updated_at": "2026-10-07T15:10:00Z",
|
||||
"language": "en",
|
||||
"phase": "brief",
|
||||
"approvals": {"requirements": {"generated": false, "approved": false}, "design": {"generated": false, "approved": false}, "tasks": {"generated": false, "approved": false}},
|
||||
"ready_for_implementation": false,
|
||||
"summary": "MCP agent + skills so developers and coding agents get the safety net's feedback early (shift-left, in-editor) and so a coding agent can be required to pass a security check before it ships. Delivery surface after G0-G5 qualification; builds on the existing ousast mcp server."
|
||||
}
|
||||
@@ -1,20 +1,5 @@
|
||||
# Design Document
|
||||
|
||||
> **Status 2026-10-04.** Groups 1-2 are done on kind (records `benchmarks/measurements/2026-10-02-k8s-01-profile/`
|
||||
> and `2026-10-02-k8s-02-store-delivery/`). Task 2.6 is paused. The production target changed. The operator runs a
|
||||
> kube-ax cluster (k3s, Agent Substrate v0.3.0, AX v0.3.1, 2 nodes). There we may only submit AX Tasks: no k8s
|
||||
> object creation, no secrets, no access to `ate-system`. The engine now runs there as AX Tasks through
|
||||
> `benchmarks/learn/engine_trace_ax.py`. The engine-task image (`plane/Dockerfile.engine-task`) is built FROM
|
||||
> `ghcr.io/norandom/ax-task-runner:v0.3.1`. Tasks read and write through presigned HTTPS links, are named
|
||||
> `ousast-engine-*`, and run a 1.7 GB heap with `ActiveProcessorCount=2` and one resident Joern session per pin. A
|
||||
> pin whose sandbox dies is retried once under a new name, then handed to the VM lane (2.5 GB heap). Egress is
|
||||
> operator-managed: HTTPS 443 to public hosts is open; LAN and cluster-internal addresses are blocked. Archive
|
||||
> names (`.tar`, `.gz`, `.zip`, `.bin`) failed through the store host's Cloudflare cache until the operator added a
|
||||
> cache-bypass rule; presigned objects are staged as `.dat` as a safeguard. Groups 3-8 are **to re-plan** against
|
||||
> that cluster under roadmap G4 (`.kiro/steering/roadmap.md`, section of 2026-10-04). Their text below is kept
|
||||
> as written. G4 (detection on push for any repository: `ousast plane scan --base --head`, engine pins on kube-ax,
|
||||
> model calls on the VM or CI) is the new target for group 7, sized by what G1-G3 keep.
|
||||
|
||||
## Overview
|
||||
|
||||
The plane keeps its shape: `ousast plane run` is the reconciler, ax is the only executor, every task is one
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"feature_name": "plane-on-kubernetes",
|
||||
"created_at": "2026-10-02T20:00:00Z",
|
||||
"updated_at": "2026-10-04T18:00:00Z",
|
||||
"updated_at": "2026-10-02T22:00:00Z",
|
||||
"language": "en",
|
||||
"phase": "implementation",
|
||||
"approvals": {
|
||||
|
||||
@@ -1,20 +1,5 @@
|
||||
# Implementation Plan
|
||||
|
||||
> **Status 2026-10-04.** Groups 1-2 are done on kind (records `benchmarks/measurements/2026-10-02-k8s-01-profile/`
|
||||
> and `2026-10-02-k8s-02-store-delivery/`). Task 2.6 is paused. The production target changed. The operator runs a
|
||||
> kube-ax cluster (k3s, Agent Substrate v0.3.0, AX v0.3.1, 2 nodes). There we may only submit AX Tasks: no k8s
|
||||
> object creation, no secrets, no access to `ate-system`. The engine now runs there as AX Tasks through
|
||||
> `benchmarks/learn/engine_trace_ax.py`. The engine-task image (`plane/Dockerfile.engine-task`) is built FROM
|
||||
> `ghcr.io/norandom/ax-task-runner:v0.3.1`. Tasks read and write through presigned HTTPS links, are named
|
||||
> `ousast-engine-*`, and run a 1.7 GB heap with `ActiveProcessorCount=2` and one resident Joern session per pin. A
|
||||
> pin whose sandbox dies is retried once under a new name, then handed to the VM lane (2.5 GB heap). Egress is
|
||||
> operator-managed: HTTPS 443 to public hosts is open; LAN and cluster-internal addresses are blocked. Archive
|
||||
> names (`.tar`, `.gz`, `.zip`, `.bin`) failed through the store host's Cloudflare cache until the operator added a
|
||||
> cache-bypass rule; presigned objects are staged as `.dat` as a safeguard. Groups 3-8 are **to re-plan** against
|
||||
> that cluster under roadmap G4 (`.kiro/steering/roadmap.md`, section of 2026-10-04). Their text below is kept
|
||||
> as written. G4 (detection on push for any repository: `ousast plane scan --base --head`, engine pins on kube-ax,
|
||||
> model calls on the VM or CI) is the new target for group 7, sized by what G1-G3 keep.
|
||||
|
||||
One task group per increment of `design.md` section 7, in that order; a later group never starts before the
|
||||
earlier group's kind proof is recorded. Every task ships with its tests; a task is done when its tests pass in
|
||||
the host suite, ruff and mypy are clean, and nothing in `src/`, `plane/`, `ops/` (minus `ops/ax/up.sh` and
|
||||
@@ -24,7 +9,7 @@ its kind proof and the record it writes under `benchmarks/measurements/<date>-k8
|
||||
(commit, images, the instrument checks, the numbers). Steps that call a model are marked **paid: needs a budget
|
||||
go-ahead** with a ceiling; steps that need a hand on a cluster that does not exist yet are marked **manual**.
|
||||
|
||||
- [x] 1. Profile and literals (design section 1; open-question checks 2, 3, 7)
|
||||
- [ ] 1. Profile and literals (design section 1; open-question checks 2, 3, 7)
|
||||
- [x] 1.1 `PlaneProfile`
|
||||
- `src/openultrasast/plane/profile.py`: frozen dataclass with the fields of design section 1; `load_profile()`
|
||||
reads `ops/k8s/profiles/<OUSAST_PLANE_PROFILE>.toml` (a name or a path; default `kind`), then applies
|
||||
@@ -122,7 +107,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
prints the table. Instrument: the tar's byte size from the store equals the extracted files' total.
|
||||
- `benchmarks/measurements/<date>-k8s-02-store-delivery/record.json`: bytes, PUT-to-poll seconds, rules.
|
||||
- _Requirements: 3.1, 3.2, 7.1_
|
||||
- [ ] 2.6 Remove the receiver (last; after 2.5 is recorded) -- paused 2026-10-04
|
||||
- [ ] 2.6 Remove the receiver (last; after 2.5 is recorded)
|
||||
- Delete `Receiver`, `_Handler`, `receiver_address`, `receiver_cluster_ip`, `RECEIVER_HOST`, `RECEIVER_PORT`
|
||||
from `egress.py`; `OUSAST_ARTIFACT_URL/DIAL/HOST/PORT` from `runner.py` and `reconciler.py`; the
|
||||
`OUSAST_DELIVERY` switch; `ops/ax/receiver-service.yaml.tmpl` and the `up.sh` receiver step; the EndpointSlice
|
||||
@@ -132,7 +117,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
`receiver_removed: <commit>`.
|
||||
- _Requirements: 3.1, 1.3_
|
||||
|
||||
- [ ] 3. `--exec remote` from a CI-shaped principal (design section 2; open-question check 1) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 3. `--exec remote` from a CI-shaped principal (design section 2; open-question check 1)
|
||||
- [ ] 3.1 Execution mode and RBAC manifests
|
||||
- `cli.py`: `ousast plane run --exec local|remote` (default `profile.exec`); `reconciler.Ax` passes the
|
||||
profile's context to `ax` (its context flag, read from `ax --help` and pinned in a test); `remote` refuses a
|
||||
@@ -165,7 +150,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
status, usd spent.
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 7.1_
|
||||
|
||||
- [ ] 4. Images on GHCR (design section 8) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 4. Images on GHCR (design section 8)
|
||||
- [ ] 4.1 `images.yml`
|
||||
- `.github/workflows/images.yml` on tags `v*`: `docker/build-push-action` for `plane/Dockerfile.runner` ->
|
||||
`ghcr.io/norandom/ousast-runner:<tag>` and the root `Dockerfile` -> `ghcr.io/norandom/ousast-engine:<tag>`;
|
||||
@@ -193,7 +178,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
- `benchmarks/measurements/<date>-k8s-04-ghcr-images/record.json`: image sizes, pull seconds, visibility.
|
||||
- _Requirements: 1.1, 4.2, 7.1_
|
||||
|
||||
- [ ] 5. The engine as a plane task (design section 4) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 5. The engine as a plane task (design section 4)
|
||||
- [ ] 5.1 `cpg/dump.py` and `tasks/engine.py`
|
||||
- Lift `benchmarks/push/finding_dump.py`'s scan into `src/openultrasast/cpg/dump.py` (the script becomes a
|
||||
thin caller; `ops/README.md:78` command unchanged). `src/openultrasast/plane/tasks/engine.py`: env
|
||||
@@ -223,7 +208,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
placement, image size and pull seconds.
|
||||
- _Requirements: 4.1, 4.2, 7.1_
|
||||
|
||||
- [ ] 6. Pools, autoscaling, node checklist (design section 5) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 6. Pools, autoscaling, node checklist (design section 5)
|
||||
- [ ] 6.1 `plane/k8s.py` and the manifests
|
||||
- `src/openultrasast/plane/k8s.py`: `render(profile, out)` writes `workerpool-default.yaml`,
|
||||
`workerpool-engine.yaml`, `ax-server-snapshots.yaml` (patch with `profile.snapshots_bucket`), `rbac-ci.yaml`;
|
||||
@@ -247,7 +232,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
- `benchmarks/measurements/<date>-k8s-06-pools/record.json`: footprint table, securityContext, dry-run result.
|
||||
- _Requirements: 5.1, 5.2, 7.1_
|
||||
|
||||
- [ ] 7. `ousast plane scan` (design section 6) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 7. `ousast plane scan` (design section 6)
|
||||
- [ ] 7.1 Workspaces and candidates from URL + commits
|
||||
- `generate.scan_cases(url, base, head, cache, *, all=False)`: shallow clone into the case cache, two pinned
|
||||
Workspaces, head-side hunks via `fix_ranges(side="new")`, functions via `repo_facts.enclosing` over
|
||||
@@ -280,7 +265,7 @@ go-ahead** with a ceiling; steps that need a hand on a cluster that does not exi
|
||||
seconds from submit to verdict, whether the known CVE site is among the agreed.
|
||||
- _Requirements: 6.1, 6.2, 6.3, 7.1_
|
||||
|
||||
- [ ] 8. Runbook (design section 7, Req 7) -- to re-plan (kube-ax, roadmap G4)
|
||||
- [ ] 8. Runbook (design section 7, Req 7)
|
||||
- [ ] 8.1 `docs/deployment.md` as the runbook
|
||||
- Sections: prerequisites for k3s (Substrate, ax with the snapshots patch, GHCR and the pull secret, buckets,
|
||||
the CI SA and its grant from 3.2, the node checklist of 6.2), the profile switch, `--exec remote` from a laptop
|
||||
|
||||
@@ -623,7 +623,6 @@ Preserve all completed task evidence below.
|
||||
- [x] 17. Operational readiness (2026-10-02)
|
||||
- **Named by the 2026-10-02 operational-readiness audit**, maintainer-approved as the first increment. The maintainer's definition: a user installs the hook on ANY repository and gets honest, advisory-only results; nothing ever blocks a push. The audit found the plumbing sound (snapshots, resolution, deadline accounting, advisory exit 0, the JSON artifact) and five things in the way: the hook can block a push, quick rules are not part of pre-push, the engine cannot finish a 26-file repository inside 30 s, the terminal hides every skip reason, and there is no supported install path without the source checkout. Before numbers for every item are the audit's live runs on records (Python), serve-static (JavaScript) and unrolled/secure (Go), each with one planted commit.
|
||||
- This group is plumbing and honesty. It enables no capability, edits no rule for a planted sample, and leaves graph reuse (the engine speed work decided in 11.6) as a later task.
|
||||
- Record: `benchmarks/measurements/2026-10-02-pre-push-operational-1/record.json` (see 17.7).
|
||||
- _Done 2026-10-02:_ all seven subtasks done. The engine still does not fit the 30 s hook deadline (graph reuse, the 11.6 direction, stays a later task), and plane verdicts after the push were not in this increment.
|
||||
- _Requirements: 9.1–9.6_
|
||||
- [x] 17.1 Never block a push the user did not choose to block
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# Brief: OpenCode-MCP proof arm (follow-up to search-with-proof)
|
||||
|
||||
Status: brief / not yet specced. Proposed 2026-10-07 by the maintainer. Builds on
|
||||
`search-with-proof` (see `../search-with-proof/verification-architecture.md`).
|
||||
|
||||
## Motivation
|
||||
|
||||
The search arm (directed, Cairn-inspired) works. The **proof-authoring** last mile is the drag:
|
||||
|
||||
1. **Safety-classifier refusals.** Authoring exploit demos repeatedly tripped a safety classifier
|
||||
this session when exploit payloads or step-by-step attack prose passed through the model or a
|
||||
tool prompt. The mitigation so far (keep payloads on disk, treat demos as opaque) is a workaround,
|
||||
not a fix, and it limits how directly the proof model can reason about the exploit.
|
||||
2. **Nondeterministic wiring.** The agent authors a demo but intermittently mis-wires the app to
|
||||
the oracle fixture (served directory for path, database URL for SQL), so a demonstrable pair
|
||||
comes back inconclusive until a retry. Each miss has cost a paid run.
|
||||
|
||||
Proposal: drive the proof with an **agentic coding tool (OpenCode MCP)** running **inside gVisor**,
|
||||
using a model chosen for exploit authoring rather than one that refuses it, and **learn** the
|
||||
proof-authoring policy with DSPy from **traces logged in OpenObserve**.
|
||||
|
||||
## Shape
|
||||
|
||||
- **Search model:** DeepSeek (current). Good at the directed fact/intent search.
|
||||
- **Proof model:** GLM 5.3 (Anthropic-recommended) and/or an abliterated model via
|
||||
`ABLITERATION_BASE_URL` (`abliterated large v2`). The abliterated model authors exploit **demos**
|
||||
without the refusals that block a safety-trained model on this defensive task.
|
||||
- **Proof driver:** OpenCode MCP (github.com/AlaeddineMessadi/opencode-mcp) as the agentic
|
||||
demo-author, running inside the gVisor verification task so the exploit-authoring is sandboxed.
|
||||
- **Learning:** DSPy compiles/optimizes the proof-authoring program (demo schema, wiring, steps)
|
||||
from past traces, instead of the per-family hand-guidance we keep adding (which is now hitting
|
||||
the prompt character budget — a signal the hand-rule approach is near its limit). Matches the
|
||||
maintainer's "learned decisions, not hand-tuning" direction.
|
||||
- **Tracing:** proof runs (reason/explore/author/verify, timings, outcomes, error classes) log to
|
||||
**OpenObserve AI** at `OPENOBSERVE_URL` (org `OPENOBSERVE_ORG`, stream `OPENOBSERVE_TRACES_STREAM`).
|
||||
DSPy consumes these traces as its training signal.
|
||||
- **Credentials:** stored in `.env` like the existing keys, referenced by name only:
|
||||
`ABLITERATION_API_KEY`, `ABLITERATION_BASE_URL`, `ABLITERATION_MODEL`, `OPENOBSERVE_URL`,
|
||||
`OPENOBSERVE_ORG`, `OPENOBSERVE_TRACES_STREAM`, `OPENOBSERVE_TOKEN`. No secret value ever enters a
|
||||
tracked file, a prompt, a log stream, or a commit.
|
||||
|
||||
## The trust-domain decision (must be resolved first)
|
||||
|
||||
The current design keeps the **model key on the VM only**; cluster tasks carry no secrets and have
|
||||
restricted egress (verify tasks reach the files host only). Running the proof **model** inside gVisor
|
||||
(as OpenCode MCP would) breaks both: the task would hold a model key and need egress to the model
|
||||
API host. Two options:
|
||||
|
||||
- **A. Scoped key + egress in the cluster (operator-managed).** Inject a spend-limited proof-model
|
||||
key into the proof task (like the existing `gemini-api-secret` slot) and add a per-host egress
|
||||
allowlist for the proof-model API. Keeps the fully-sandboxed agentic authoring the maintainer
|
||||
wants; requires an operator change and widens the cluster's secret/egress surface.
|
||||
- **B. Brain on the VM, execution in gVisor (current split preserved).** OpenCode MCP and the proof
|
||||
model run on the VM; only the DAST execution stays in gVisor. No new cluster secret or egress;
|
||||
but the agentic authoring loop is not itself sandboxed.
|
||||
|
||||
Recommendation: start with **B** (no trust-domain change, fastest to try the abliterated proof model
|
||||
and DSPy loop), and move to **A** only if sandboxing the authoring loop itself proves necessary.
|
||||
|
||||
## Security scope (why an abliterated model is in-scope here)
|
||||
|
||||
This is authorized **defensive** work: authoring proof-of-concept demos for **known CVE fix-pairs**
|
||||
in a **gVisor-isolated** corpus, to demonstrate a regression before push. Guardrails to keep:
|
||||
|
||||
- Demos are **data**, not agent code; they stay in the gitignored private area, never echoed.
|
||||
- No real-world or third-party live targets; only the frozen fix-pair corpus and owned oracles.
|
||||
- gVisor is the execution boundary; the abliterated model never runs code itself, it only writes a
|
||||
declarative demo the trusted verifier executes.
|
||||
- The abliterated model is confined to proof-authoring; the search and all reporting stay on the
|
||||
normal models.
|
||||
|
||||
## Open questions for the spec
|
||||
|
||||
1. A or B for the trust domain (above).
|
||||
2. Does the abliterated proof model actually raise the demonstrated rate and remove the wiring
|
||||
misses, measured on the pairs already shown demonstrable (xiaomusic, glance)? Qualify before scaling.
|
||||
3. DSPy program shape: what is the proof-authoring signature (inputs: candidate + oracle contract;
|
||||
output: a valid, wired demo), and what metric does it optimize (demonstrated rate at zero false
|
||||
proofs) from OpenObserve traces?
|
||||
4. Does OpenCode MCP fit the existing executor/verifier task contract (no-secret, presigned I/O,
|
||||
scratch/egress limits), or does it need a new task kind?
|
||||
5. Cost envelope per proof vs the current DeepSeek authoring.
|
||||
|
||||
## OpenCode MCP as the endpoint for both PoC and fix
|
||||
|
||||
Maintainer note (2026-10-07): OpenCode MCP is a good server endpoint for the PoC **and** the fix,
|
||||
not only proof authoring. The same agentic endpoint can:
|
||||
|
||||
1. Author the PoC (the demonstrated proof) for a caught finding.
|
||||
2. Write a candidate fix against the finding's repair direction and the failing PoC.
|
||||
3. Hand the fix back to the differential verifier, which re-runs the PoC and confirms absence
|
||||
(the proof flips from failing to passing).
|
||||
|
||||
This turns the "no auto-patch" gap in `../../../docs/use-case-fix-and-reverify.md` into "the proof
|
||||
endpoint also proposes the fix, and the fix is accepted only when the proof it generated goes green."
|
||||
The PoC is the acceptance test for the fix, so a wrong fix is caught, not trusted. The fix stays a
|
||||
proposal for the developer to review; the verifier's green proof is the objective signal, not the
|
||||
model's say-so. This fix capability is specced alongside the proof authoring when this arm is built.
|
||||
|
||||
## Not in scope
|
||||
|
||||
Changing the search arm, the oracle differential, or the abstract-interpretation triage. This brief
|
||||
is about the **proof-authoring and fix** driver and its learning loop, planned as a full spec after
|
||||
the safety net is qualified.
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"feature_name": "proof-arm-opencode",
|
||||
"created_at": "2026-10-07T14:40:00Z",
|
||||
"updated_at": "2026-10-07T14:40:00Z",
|
||||
"language": "en",
|
||||
"phase": "brief",
|
||||
"approvals": {"requirements": {"generated": false, "approved": false}, "design": {"generated": false, "approved": false}, "tasks": {"generated": false, "approved": false}},
|
||||
"ready_for_implementation": false,
|
||||
"summary": "OpenCode-MCP proof-authoring arm for search-with-proof: abliterated/GLM proof model, DSPy-optimized from OpenObserve traces; trust-domain decision (A scoped-key-in-cluster vs B brain-on-VM) must be resolved first. Creds in .env by name."
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
# Search-with-proof — handoff (2026-10-06)
|
||||
|
||||
State at handoff, for resuming on the search problem and abstract interpretation.
|
||||
|
||||
## RESOLVED 2026-10-07 — the one open question is CLOSED: first cluster demonstration achieved
|
||||
|
||||
The "first `demonstrated`" question below is answered. search-with-proof produced its first autonomous
|
||||
`demonstrated` finding end to end on kube-ax, and a blind negative confirmed no false proof.
|
||||
|
||||
- **Pair:** hanxi/xiaomusic (pilot pair 13), family `path`, vulnerability class **prefix-check bypass**
|
||||
(`startswith(root)` without a trailing separator; the fix adds `os.sep`). Image `e2c8b460`.
|
||||
- **Vulnerable side:** `demonstrated` — owned effect in 3/3 affected runs, 0/3 safe runs. $0.37, 55 calls.
|
||||
- **Fixed side (blind negative, Req 6.3):** `inconclusive`, **0 false proofs**. $0.56, 85 calls.
|
||||
- **Pair total:** $1.22. Record: `benchmarks/measurements/2026-10-07-search-first-cluster-demonstration/`.
|
||||
|
||||
**The original pair-6 both-sides-false was a TWO-part diagnosis, both now fixed:** the demo did not reach the
|
||||
observable (demo/agent), and — the decisive one — the path oracle only modelled **classic parent traversal** while
|
||||
many real path CVEs (xiaomusic included) are **prefix-bypass**, which reaches a sibling that string-extends the served
|
||||
dir, not the parent. The oracle now also plants a prefix-sibling canary.
|
||||
|
||||
**Fix chain that enabled the demonstration (each merged to main and validated by a paid run):**
|
||||
1. Parity — agent is told the observable's location (path oracle reached parity with command/ssrf).
|
||||
2. Verify-time-only — the observable is injected at verify time and absent during exploration; stop the explore-time
|
||||
reachability trap that burned budget without authoring a demo.
|
||||
3. Fair-share pilot budget — each search gets a fair share of `--ceiling-usd` with rollover, so one expensive search
|
||||
no longer starves the rest.
|
||||
4. Served-root guidance — the demo MUST set the app's served directory to the `${served_root}` placeholder.
|
||||
5. Oracle prefix-bypass coverage — plant a sibling canary for the `startswith`-without-separator class.
|
||||
6. Control-signal feedback — the keystone control (does the app serve `${served_root}`?) is fed back to the agent as a
|
||||
`served-root` hint so it self-corrects its wiring. This was the decisive last step.
|
||||
|
||||
**Keystone (honest measurement), merged earlier:** the verifier runs a benign positive control; a broken or mis-wired
|
||||
app yields `could_not_run`/annotation, never a silent clean-looking `inconclusive`.
|
||||
|
||||
**Next toward the pilot exit (Req 6.2):** fan out the path family to ≥5 demonstrations with 0 false proofs, then the
|
||||
≥73-negative BLOCK qualification (Req 6.3). The glance pair (classic traversal) also demonstrates once its npm
|
||||
dependency build-break (`filed`→`mime@4`) is pinned.
|
||||
|
||||
---
|
||||
|
||||
## The one open question (HISTORICAL — resolved above)
|
||||
|
||||
The full chain runs end to end on kube-ax. Every execution layer is cleared. The **first `demonstrated` has not
|
||||
happened yet**, and the remaining gap is narrow and reproducible at **$0 model cost** via `demo_replay`.
|
||||
|
||||
No-model replay of pilot pair 6 (path / Node / npm build), both sides, image `aa54880`:
|
||||
- Both sides **build, start, and run** (ready ~18–20 s, "fresh task run completed", no `could_not_run`).
|
||||
- Oracle observed **no effect on either side** → `inconclusive`, "no consistent differential".
|
||||
|
||||
Both-sides-false means it is **not the harness** any more. It is one of:
|
||||
1. The demo's attack step as written does not trigger the effect even on the vulnerable revision, or
|
||||
2. The oracle does not capture the app's output where the effect would show.
|
||||
|
||||
## How to close it (no model spend)
|
||||
|
||||
- Control: the **probe path app** (`src/openultrasast/search/probe_apps/`) is known-good and *does* observe through the
|
||||
same verifier code path. Diff its behaviour against the real app: how the served directory is wired
|
||||
(`${served_root}` substitution in `start.arguments`), and where the oracle reads the nonce from (app stdout/response
|
||||
capture in `oracles.py` `PathOracle.observe`).
|
||||
- Re-run with `python -m benchmarks.search.demo_replay --both --demo <stored demo.json> --family path --lane ax
|
||||
--image <digest> ...`. Stored demos live under `benchmarks/search/private/runs/*/<pair>/demo/demo.json` (gitignored).
|
||||
- **Classifier note:** keep exploit payloads on disk. Hand Codex file-path tasks that treat demo contents as opaque;
|
||||
print outcomes / phase timings / error *classes*, never request strings, env values, or step bodies. See memory
|
||||
`search-with-proof-direction` and the classifier false-positive note. Code changes never tripped it; echoing
|
||||
payloads or step-by-step attack prose did.
|
||||
|
||||
## What is done and merged (main)
|
||||
|
||||
- Trust-domain split: model key on the VM only; executor and verifier tasks carry no secrets; exchange via presigned
|
||||
objects. Declarative demos (data, not agent code). gVisor is the cluster boundary.
|
||||
- Verifier + 5 oracles pass inside a gVisor task (probe: 5/5 demonstrated, gaming demos inconclusive).
|
||||
- Shared task base image, code-last layers; `plane/task-base.digest` pins it; `engine-image.yml` builds only on
|
||||
dispatch / tags / Dockerfile / pin changes. Measured unpacked: search 2.07 GB, engine 3.61 GB.
|
||||
- Pilot driver (`benchmarks/search/pilot_run.py`), budgets with reservation, named exhaustion limits, `demo_replay`.
|
||||
- Per-host egress live: exec → any 443; verify/probe/pass → files host; replay → files + github. Task names carry
|
||||
the right prefixes.
|
||||
|
||||
## Plumbing defects fixed along the way (each cost one run)
|
||||
|
||||
step allowance → family-vs-oracle mapping → build memory (256 MB SIGTRAP) → manifest path appended positionally →
|
||||
startup env + data files → dead-end intent dedupe → archive size (edge ~100 MB, now split+gzip) → two-interpreter
|
||||
image (pip 3.13 vs venv 3.12 → use `sys.executable`) → checkout root on sys.path → `node_modules` path.
|
||||
|
||||
## Next, once a demo demonstrates
|
||||
|
||||
1. Fan out the pilot across all 20 pairs, both sides (feasibility: demonstrated rate per family, zero false proofs on
|
||||
fixed sides). Exit: ≥1 family with ≥5 demonstrations and zero false proofs (Req 6.2).
|
||||
2. BLOCK qualification per family: 0 false proofs over ≥73 negatives from ≥73 repos (Req 6.3).
|
||||
3. G1 arm on the unseen pool vs the triage/decision arms at equal false-alarm budgets (Req 6.5).
|
||||
|
||||
## Cluster / spend facts
|
||||
|
||||
- kube-ax: **2 tasks in flight, one per worker** (operator `--max-actors=1`). Dispatcher caps AX concurrency at 2;
|
||||
raise it only when the operator raises worker count. "no free workers available" = retry, not failure.
|
||||
- `/workspace` ~3 GiB free per node until the volume move; scratch guard 2 GiB/task.
|
||||
- Pilot spend to date ~$11 across 9 paid runs; the decisive replay was $0.
|
||||
|
||||
## Decision model vs search — current read
|
||||
|
||||
- Decision/classification: measured, **promising as triage** (within-pair AUC 0.762; trained logistic ties at 0.721),
|
||||
**capped as a blocker** (cannot prove 95% precision). Reframed as triage (`learned-decision-engine` Req 6.5).
|
||||
- Search-with-proof: **not yet demonstrated**, but no observation so far argues against the idea — every failure was
|
||||
our execution, now cleared. The pair-6 demo/oracle question is the first verdict on the method itself.
|
||||
@@ -1,40 +0,0 @@
|
||||
# Brief: search with proof
|
||||
|
||||
Maintainer, 2026-10-05, after reviewing Cairn (github.com/oritera/Cairn): "we are still not progressing through the
|
||||
state space search / not finding what we need and the progress is sort of slow ... i think something is missing."
|
||||
And: "I'd move to primarily executing in the cluster."
|
||||
|
||||
## What was missing
|
||||
|
||||
Every detection step so far classifies: it scores one function excerpt and then tries to make the score precise
|
||||
enough to block. Calibration, Wilson bounds, pooled block gates, prompt variants and trained models all hit the same
|
||||
wall: ranking is decent (within-pair AUC about 0.76), but precision at the top cannot be proven to 95% on our data.
|
||||
|
||||
Cairn treats a security problem as a directed search with an objectively checkable goal. A shared board holds
|
||||
confirmed facts, open intents and human hints. A reasoner proposes independent intents; explorers with real tools
|
||||
each confirm one fact; the loop ends when the goal is demonstrably met.
|
||||
|
||||
Three things follow for OpenUltraSAST:
|
||||
|
||||
1. **An objective success condition.** A finding is demonstrated when a test or proof of concept fails on the
|
||||
vulnerable code and passes on the fixed code (or, for a push, fails on head and passes on base where applicable).
|
||||
A demonstrated finding is right by construction; it needs no calibration and no training data, so it generalises
|
||||
to unseen repositories by construction.
|
||||
2. **Exploration with tools.** Agents that read other files, follow callers, build and run the code, instead of a
|
||||
single-shot judgement of one excerpt.
|
||||
3. **Search state per repository and push.** Facts and intents accumulate on a board instead of every question
|
||||
starting from zero.
|
||||
|
||||
## Shape
|
||||
|
||||
- The cheap layers (quick rules, engine, classifier) become triage: they rank candidates.
|
||||
- Agentic search runs only on the top-ranked candidates of a push, under a per-push budget.
|
||||
- BLOCK only on a demonstrated finding. Advisory for plausible, unproven ones. "Could not demonstrate" is recorded as
|
||||
such, never as "not vulnerable".
|
||||
- Execution is primarily in the kube-ax cluster: reason and explore steps are AX Tasks; the VM coordinates.
|
||||
- Model access: a scoped, budget-limited key in the cluster, managed by the operator.
|
||||
|
||||
## Not in scope
|
||||
|
||||
Copying Cairn code (AGPL-3.0; this repository is Apache-2.0). Offensive use against systems we do not own: all
|
||||
execution is against code under analysis, inside gVisor sandboxes.
|
||||
@@ -1,25 +0,0 @@
|
||||
# Design: search with proof (v2, after two Codex reviews; the Claude Doc is the readable version)
|
||||
|
||||
## Why: from scoring excerpts to proving findings
|
||||
BLOCK will mean a vulnerability was demonstrated and re-run by the system, not that a score passed a threshold. Every detection step so far classified one function excerpt, then tried to make the score precise enough to block. Same wall in every experiment: LLM programs (exp-005 A) within-pair AUC 0.762, pooled gate needs >=73 clean flags; trained logistic 0.721 (tie, CI spans zero); gradient boosting 0.597 (worse). Cairn (AGPL, design only) treats a security problem as directed search with a checkable goal. Three things follow: (1) objective goal: test/PoC fails on vulnerable, passes on fixed; needs no calibration or training data, so it carries over to unseen repos; (2) exploration with tools (read other files, follow callers, build, run); (3) search state: facts and intents accumulate on a board per search.
|
||||
|
||||
## Architecture
|
||||
VM coordinates and is the only writer to the board (memory store); every model call and execution runs as an AX Task on kube-ax. Triage (quick rules, engine, classifier) -> top K candidates -> first facts on the board. Reason task (reads snapshot, returns complete or <=3 intents) <-> board; Explore tasks (one per intent; read, grep, run, write demo/; return one fact) <-> board; Verify task (fresh sandbox, planted canary, vulnerable vs fixed / head vs base) -> outcome: BLOCK only if demonstrated, else advisory or silent. Scheduler and budgets on the VM check caps before every dispatch. Tasks read/write via presigned links; store credentials stay on the VM. VM is fallback lane, kind local fallback, same image.
|
||||
|
||||
## The demonstration contract (v2)
|
||||
The verifier, not the agent, observes the effect, on one side and not the other, consistently, in fresh sandboxes. Artefact: demo/ with build.sh and attack.sh (or a request file) touching only the documented external interface; its exit code and output are ignored. Oracles owned by the verifier per family: injection SQL = canary row seen in verifier's query log/response capture; path = canary file outside root read, seen by file-access watch; XSS = script executes in verifier's headless browser; SSRF = request arrives at verifier's in-sandbox listener; injection (command) = canary side effect seen by process watch. Deserialisation, access control, config/secrets: no owned oracle yet -> plausible_unproven at most. Attacker boundary excludes source tree, dependencies, runtime, fixtures; checkout and dependencies read-only during run; untracked files, dependency swaps, runtime patching refused. Differential: fix pair vulnerable-vs-fixed; push head-vs-base. Both sides must build and pass readiness independently; each side 3 runs from fresh state; inconsistent runs rejected; a side that fails to build or start => inconclusive, never a pass. Outcomes: demonstrated (BLOCK once the family is qualified), plausible_unproven (advisory, a visible finding), could_not_build/could_not_run/inconclusive (advisory if facts support, else silent; never "not vulnerable"), no_oracle (as plausible_unproven), no_evidence (silent). REGRESS's nonzero-exit-is-triggerable rule (regress/verdict.py) is not reused for proof; its sandbox plumbing is.
|
||||
|
||||
## Agent worker and sandbox tools
|
||||
Own small tool loop in src/openultrasast/search/ via the existing OpenAI-compatible client (provider swappable, spend metered, Apache-2.0). Image ousast-search-task from the AX task runner, toolchains for Python, Node and TypeScript, PHP, Java; repo fetched over HTTPS. Tools: list_files, read_file (400 lines/call), grep (200 matches), run (per-call timeout, truncated output), write_demo (only write, under demo/), finish (one fact or recorded failure). Engine evidence (taint path, sinks, missing guards) is in the explore prompt. Reason step uses no tools.
|
||||
|
||||
## Triage, budgets and model access (v2)
|
||||
Top K per push (K=3); candidate carries family hint + engine evidence as first facts. Caps (ceilings, not estimates): reason rounds 4/search, intents 3/round, explore 15 min/intent, $0.50/search, $1.50/push, 3 demos x 2 sides x 3 runs. Spend reserved before each model call at max cost and settled after (plane/budget.py gets a reservation step); per-task memory/disk/cleanup limits sized to workers (~3 vCPU, 5-5.75 GiB, two heavy lanes); total run wall-time and retry cap. Worst case 120 task-hours for 40 searches (~60 h on two lanes); a 1,300-change slice at K=3 could reach $1,950. First 3 pilot searches measure real cost; pilot ceiling set from that and asked for. Model key: operator scoped secret (AX injects only GEMINI_API_KEY; operator decides mechanism); never in manifest/log/board/record; repository code cannot read it: model client runs in a separate process as its own user.
|
||||
|
||||
## Pilot, qualification, then the unseen pool (v2)
|
||||
1. Feasibility pilot: 20 dev-corpus pairs (never v3/pool), the four pool families with an owned oracle (injection, path, output encoding, untrusted destination), across the pool's five languages (Python, JS, TS, PHP, Java). Candidate = labelled function (measures proof, not triage); advantages recorded; fixed-side search never sees the vulnerable revision or its demo. Reported: demonstrated rate per family/language, false proofs, could_not_build/run/inconclusive/no_oracle rates, spend/tasks/wall. Exit: >=1 family with >=5 verified demonstrations and zero false proofs, else stop.
|
||||
2. BLOCK qualification per family: 0/20 still allows 16.1% (Wilson 95%); a family BLOCKs only after >=73 negative-side searches (fixed revisions + ordinary changes) with zero demonstrated (upper bound 5%, same bar as the pooled gate). Until then demonstrations show as advisories.
|
||||
3. G1 arm on the unseen pool: triage top K on every replayed change in a fresh slice; scored with the pool's rules on every visible finding incl. advisories, family + location match; 1/5/10% FA budgets beside classifier arms; adopted only under the no-regression rule. No evaluation label, fixing revision or evaluation board is ever an input to unseen search.
|
||||
Every run reports the coverage ceiling (true candidate x owned oracle x both sides runnable x genuine differential, build failures in the denominator) next to catch. Delayed results go via push/background.py and are scored separately from push-time results.
|
||||
|
||||
## Decisions, risks and open questions
|
||||
Largest risk: environment failure caps the demonstrated rate. Risks: builds fail; agent games oracle (mitigated by re-run, canary, differential, tracked-file check); repo code misuses egress (gVisor; ask operator for no-egress verify class); cost (hard caps); minutes per search too slow for synchronous hook (decide product path after pilot). Open: operator key mechanism + no-egress class; own tool loop vs CLI agent; pilot go-ahead $20; how a minutes-long search reaches the pre-push user.
|
||||
@@ -1,80 +0,0 @@
|
||||
# Declarative verification and separate execution domains
|
||||
|
||||
Implemented on `search-with-proof`, 2026-10-05, under the maintainer's explicit instructions.
|
||||
`git merge -q main` failed because the shared Git metadata is read-only. The amended requirements
|
||||
were read from the main checkout, including Requirements 3.1 and 5.2. No commits or network operations.
|
||||
|
||||
## Changes
|
||||
|
||||
- Strict, bounded JSON demonstration schema, also supplied to the model's `write_demo` tool.
|
||||
Verifier-owned recipes cover pip, npm, composer, Maven, Gradle and no build. Runtime entrypoints,
|
||||
CLI/HTTP sequences and earlier-response captures are validated; private fixtures, environment
|
||||
substitutions, canary captures and script artefacts are rejected. Injection/traversal strings remain
|
||||
legitimate request payload data, never executable verifier instructions.
|
||||
- The verifier owns build, readiness and request execution. Local runs retain Bubblewrap (`userns`
|
||||
on this host). HTTP requests require an acknowledged readiness gate; incomplete observations fail
|
||||
closed. Build-generated internal symlinks are supported without accepting external or cyclic links.
|
||||
- A supplied AX side dispatcher selects `task-boundary` and creates three separate fresh tasks per
|
||||
side. Failure of local isolation without a dispatcher fails closed.
|
||||
- The model-holding worker delegates list/read/grep/run to an executor interface. Production uses
|
||||
presigned command/result objects; the in-process adapter is for tests. Commands have sequence
|
||||
numbers, identical-retry caching, deadlines, stop and bounded results. Uncertain outcomes prevent
|
||||
later commands from reusing that executor session.
|
||||
- AX executor tasks use the `ousast-engine-search-exec-` prefix. Task entrypoints reject inherited
|
||||
credentials before reading checkout input, then use an explicit environment allowlist.
|
||||
|
||||
## Changed files
|
||||
|
||||
Production:
|
||||
- `src/openultrasast/search/demo.py`
|
||||
- `src/openultrasast/search/executor.py`
|
||||
- `src/openultrasast/search/worker.py`
|
||||
- `src/openultrasast/search/verify.py`
|
||||
- `src/openultrasast/search/verify_task.py`
|
||||
- `src/openultrasast/search/_sandbox.py`
|
||||
- `src/openultrasast/search/oracles.py`
|
||||
- `src/openultrasast/search/probe.py`
|
||||
- `src/openultrasast/search/probe_apps/app.py.txt`
|
||||
- `benchmarks/ax/batch.py`
|
||||
- `benchmarks/ax/search_verify.py`
|
||||
- `plane/Dockerfile.search-task`
|
||||
|
||||
Tests and audit:
|
||||
- `tests/test_search_demo.py`
|
||||
- `tests/test_search_executor.py`
|
||||
- `tests/test_search_verify_transport.py`
|
||||
- `tests/test_search_verify.py`
|
||||
- `tests/test_search_worker.py`
|
||||
- `tests/test_search_isolation.py`
|
||||
- `tests/test_search_probe.py`
|
||||
- `tests/test_ax_batch.py`
|
||||
- `benchmarks/measurements/2026-09-08-module-audit.json`
|
||||
|
||||
This report is the additional documentation file.
|
||||
|
||||
## Fresh validation
|
||||
|
||||
`/home/mc/Source/OpenUltraSAST/.venv/bin/pytest -q tests/test_search_*.py tests/test_ax_batch.py tests/test_plane_literals.py tests/test_module_audit.py -rs`
|
||||
|
||||
Result: **171 passed, 3 skipped, exit 0**, in 35.46 seconds.
|
||||
The three skips are Chromium isolation/network setup, SSRF localhost sockets and HTTP localhost sockets,
|
||||
all prohibited by this host's policy. Host verification mode was explicitly checked: `userns`.
|
||||
|
||||
Ruff passes for search sources, AX changes and affected tests. Targeted strict mypy passes for the seven
|
||||
changed Python search implementation modules checked. Independent review approved the implementation,
|
||||
including follow-up timeout, readiness, capture, credential and generated-link regressions.
|
||||
|
||||
Module audit: **171 modules**, **165 load-bearing**, **6 standalone capabilities**, **0 orphaned**.
|
||||
The audit read the worktree source explicitly with `PYTHONPATH=src`.
|
||||
|
||||
## Remaining validation gaps
|
||||
|
||||
- No live AX/gVisor/kind dispatch, image build/pull/publish, or paid model calls were performed.
|
||||
- Actual ecosystem dependency installations remain unqualified. Python dependency loading was tested
|
||||
with a controlled offline build output and real isolated execution.
|
||||
- The three socket/browser integrations need a host or task where those operations are permitted.
|
||||
- Hard aggregate disk isolation in `task-boundary` mode requires an outer task storage quota. The
|
||||
executor records this requirement and additionally bounds individual files and monitors checkout
|
||||
growth; it cannot enforce a quota across every writable task path without that outer resource limit.
|
||||
- AX tasks configured to inject a model credential will fail closed; executor and verifier task classes
|
||||
must be provisioned without that injection.
|
||||
@@ -1,101 +0,0 @@
|
||||
# Host launcher repair and kube-ax operator constraints
|
||||
|
||||
2026-10-05, branch `search-with-proof`. Implemented under the maintainer's explicit
|
||||
instructions. No commits or network operations. Existing uncommitted work was preserved.
|
||||
This supersedes the build/transport and disk-bound statements in
|
||||
`implementation-trust-domains.md` where they differ.
|
||||
|
||||
## Root cause and repair
|
||||
|
||||
The HTTP driver calls `subprocess.Popen(..., stdin=DEVNULL, stdout=DEVNULL,
|
||||
stderr=DEVNULL)` inside bubblewrap. Its filesystem had no `/dev/null`; Python failed
|
||||
in `_get_handles` before starting the app, binding port 18765, or applying captures.
|
||||
Other probe apps do not use this nested HTTP launcher path.
|
||||
|
||||
A socket-free before/after reproduction read a 19-byte instrument input. Removing
|
||||
`--dev /dev` reproduced exit 1 and the reported `subprocess.py:992 → _get_handles`
|
||||
stack, ending with `FileNotFoundError: [Errno 2] No such file or directory: '/dev/null'`.
|
||||
With the private device mount, the identical launcher exited 0 with empty stderr.
|
||||
The original HTTP capture test's fixed port, token exchange, and differential
|
||||
assertion remain unchanged. Reasons now retain the last 2,000 stderr characters;
|
||||
executor result truncation retains stderr tails too.
|
||||
|
||||
## Operator contract
|
||||
|
||||
- Shared two-slot AX reservations cover workload lane instances and long-lived
|
||||
search executors. Capacity rejection retries the identical manifest with 15–30
|
||||
seconds of jitter until the deadline. Exhaustion is `capacity_timeout`, never
|
||||
`sandbox_failure`, and does not trigger another execution attempt or VM fallback.
|
||||
Uncertain task cleanup retains the reservation and halts dispatch.
|
||||
- Task entrypoints configure temporary files and the specified package caches under
|
||||
`/workspace`. The aggregate guard defaults to 2 GiB, configurable through
|
||||
`OUSAST_SCRATCH_BYTES`. It checks workspace files, caches, and unlinked output logs
|
||||
during child execution and after exit. Exceeding it returns `could_not_build` with
|
||||
`scratch limit`; unreadable scratch is not interpreted as zero usage.
|
||||
- The default AX side dispatcher first builds/packages each side in an executor,
|
||||
retrieves its digest-checked archive, and deletes the executor before verification.
|
||||
One archive contains the checkout, products, and spec. Three fresh verifier tasks
|
||||
consume that archive through `VERIFY_INPUT_URL`; results use `RESULT_URL`.
|
||||
Verifiers reject dependency builds. Git metadata is omitted from source archives;
|
||||
safe internal generated links are materialized before export.
|
||||
- Search tasks receive source archives. Executor tool guidance and project/operator
|
||||
documentation require HTTPS GitHub git clone or codeload, never `api.github.com`.
|
||||
First-download retries tolerate delayed egress activation for up to 60 seconds.
|
||||
|
||||
## Files changed in this follow-up
|
||||
|
||||
Production:
|
||||
- `src/openultrasast/search/_sandbox.py`
|
||||
- `src/openultrasast/search/verify.py`
|
||||
- `src/openultrasast/search/executor.py`
|
||||
- `src/openultrasast/search/verify_task.py`
|
||||
- `src/openultrasast/search/task_storage.py` (new)
|
||||
- `src/openultrasast/search/worker.py`
|
||||
- `benchmarks/ax/batch.py`
|
||||
- `benchmarks/ax/search_verify.py`
|
||||
|
||||
Tests:
|
||||
- `tests/test_search_isolation.py`
|
||||
- `tests/test_search_verify.py`
|
||||
- `tests/test_search_verify_transport.py`
|
||||
- `tests/test_search_task_storage.py` (new)
|
||||
- `tests/test_ax_batch.py`
|
||||
|
||||
Documentation/evidence:
|
||||
- `.kiro/steering/kube-ax.md` (new)
|
||||
- `ops/ax/README.md`
|
||||
- `benchmarks/measurements/2026-09-08-module-audit.json`
|
||||
- This report (new).
|
||||
|
||||
Other files already modified/untracked at session start remain uncommitted.
|
||||
|
||||
## Fresh verification
|
||||
|
||||
- Requested suite, with `-rs` for skip reasons:
|
||||
`/home/mc/Source/OpenUltraSAST/.venv/bin/pytest -q tests/test_search_*.py tests/test_ax_batch.py tests/test_plane_literals.py -rs`
|
||||
— **181 passed, 3 skipped, exit 0**, 24.94 seconds.
|
||||
- Final executor/storage regression and module audit tests after tightening stderr
|
||||
truncation: **34 passed, exit 0**, including all **6 module audit tests**.
|
||||
- Shared dispatcher consumer `tests/test_unseen_calibrate.py`: **10 passed, exit 0**.
|
||||
- Module audit regenerated from the worktree using `PYTHONPATH=src`, with an explicit
|
||||
source read/byte count: **172 modules, 166 load-bearing, 6 standalone capabilities,
|
||||
0 orphaned**.
|
||||
- Ruff passes across search, AX, and changed tests; strict mypy passes for the six
|
||||
changed typed implementation modules. `git diff --check` passes.
|
||||
|
||||
## Gaps
|
||||
|
||||
- Host policy prohibits Chromium network/isolation setup, SSRF localhost sockets,
|
||||
and the full HTTP capture test. These are the three skips. The exact missing-device
|
||||
failure is reproduced and repaired without sockets; full HTTP host confirmation
|
||||
remains necessary.
|
||||
- No live AX tasks, image builds/pulls, registry installs, or external downloads were
|
||||
run. Mocked transport plus local real child processes establish code contracts,
|
||||
not deployed cluster readiness. Verifier egress to
|
||||
`files.because-security.com:443` is requested from the operator and still pending.
|
||||
- Scratch enforcement is a polling guard, not an instantaneous filesystem quota.
|
||||
Programs that hardcode RAM-backed paths can still consume the worker memory limit.
|
||||
Archive bounds remain 64 MiB compressed / 256 MiB file contents / 20,000 entries;
|
||||
larger built projects fail explicitly and require a separately qualified envelope.
|
||||
- Multi-ecosystem dependency build/runtime qualification remains open. The offline
|
||||
preparation tests cover no-build projects and controlled Python build products.
|
||||
@@ -1,146 +0,0 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
Add a search-with-proof stage: for the top-ranked candidates of a push or a labelled pair, sandboxed tool-using agents
|
||||
search for a demonstration of the vulnerability, coordinated through a fact/intent board, executing primarily in the
|
||||
kube-ax cluster. BLOCK becomes "demonstrated", not "scored above a threshold". Context: `brief.md`, roadmap G1-G2,
|
||||
`unseen-repo-evaluation` (the pool and metric this arm is judged by).
|
||||
|
||||
## Boundary Context
|
||||
|
||||
- **In scope**: the board (facts, intents, hints) in the memory store; reason and explore steps as AX Tasks; the
|
||||
demonstration contract and its verification; triage-to-search budgeting; a pilot on known fix pairs; a G1 arm on the
|
||||
unseen pool; model access through an operator-managed scoped key.
|
||||
- **Out of scope**: copying Cairn code (AGPL-3.0); network actions against anything but the code under analysis;
|
||||
changing the existing quick rules, engine or classifier (they are inputs here).
|
||||
- **Adjacent expectations**: `unseen-repo-evaluation` scores this arm; the AX batch dispatcher (`benchmarks/ax/batch.py`)
|
||||
and the kube-ax constraints (2 heavy lanes, `ousast-engine-` names, egress to public HTTPS) apply;
|
||||
`learned-decision-engine` Req 6.4 BLOCK rule is superseded for this arm by "demonstrated".
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: The board
|
||||
|
||||
**User Story:** As the maintainer, I want each search to keep its state, so exploration builds on what is already
|
||||
known instead of restarting.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. Per search (one candidate or one push), the board holds facts (confirmed, objective, with the evidence that
|
||||
confirmed them), intents (declared directions, open or concluded, with the facts they start from) and hints (human
|
||||
input, absorbed on the next read).
|
||||
2. The board lives in the memory store, versioned, and every fact and intent carries the step and the task that wrote
|
||||
it.
|
||||
3. A search ends when a fact meets the goal (Requirement 3), when its budget is spent, or when the reasoner proposes no
|
||||
new intent and none is open; the end reason is recorded.
|
||||
|
||||
### Requirement 2: Reason and explore steps
|
||||
|
||||
**User Story:** As the maintainer, I want the search driven by the board's state, not by fixed roles.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. A reason step reads the board and returns: the goal is met (citing facts), or at most N new, independent,
|
||||
non-overlapping intents, or no new intent.
|
||||
2. An explore step claims one intent, works it with tools inside the sandbox (read files, search the repository,
|
||||
follow callers, build, run tests and scripts against the code under analysis), and returns exactly one new fact or
|
||||
a recorded failure.
|
||||
3. Both run as AX Tasks in the kube-ax cluster by default; the VM coordinates (board, dispatch, budgets) and is a
|
||||
fallback lane only.
|
||||
4. Parallel explores of one search never race on the board (claims as in the batch dispatcher).
|
||||
|
||||
### Requirement 3: The demonstration contract
|
||||
|
||||
**User Story:** As the maintainer, I want BLOCK to mean "shown", so precision does not depend on calibration.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. *(Amended 2026-10-05, maintainer decision after the kube-ax probe: AX tasks have no capabilities, so gVisor and one
|
||||
fresh task per side are the only isolation.)* A demonstration is **declarative data, never agent-written code**: a
|
||||
build recipe chosen from the verifier's fixed per-ecosystem recipes, a start command of the form runtime + path +
|
||||
arguments (no shell), and a request sequence (HTTP requests or CLI invocations, with values captured from earlier
|
||||
responses) that the verifier itself sends. The system, not the agent, runs it in fresh sandboxes. The **verifier observes the effect itself** through a per-family oracle it owns (for
|
||||
example a request log, a planted resource it watches, a browser execution check for XSS); the artefact's own exit
|
||||
code or output is never the oracle.
|
||||
2. Each family's oracle states the attacker boundary (what the artefact may touch: only the documented external
|
||||
inputs of the code under analysis, never the verifier's fixtures, the source tree, dependencies or the runtime).
|
||||
Inputs are immutable during a run; untracked files, dependency replacement and runtime patching are refused.
|
||||
3. Differential: for a labelled pair the effect is observed on the vulnerable revision and not on the fixed one; for a
|
||||
push, on head and not on base. Both sides must build and pass a readiness check independently; a side that fails
|
||||
to build or start makes the result **inconclusive**, never a pass. Each side is run at least three times from fresh
|
||||
state; inconsistent runs are rejected.
|
||||
4. Only a verified demonstration counts as `demonstrated`. Outcomes: `demonstrated`, `plausible_unproven`,
|
||||
`could_not_build`, `could_not_run`, `inconclusive`, `no_oracle` (family without an owned oracle), `no_evidence`;
|
||||
none of the failure outcomes is ever reported as "not vulnerable".
|
||||
5. Families without an owned oracle (for now: deserialisation, access control, configuration and secrets) can reach
|
||||
`plausible_unproven` at most until an oracle is specified and reviewed.
|
||||
|
||||
### Requirement 4: Triage and budget
|
||||
|
||||
**User Story:** As the maintainer, I want search spent where it matters, at a known cost.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. Candidates come from the existing layers (quick rules, engine, classifier scores); only the top K per push or per
|
||||
pair side are searched; K and the per-search and per-push budgets (model spend, task count, wall time) are
|
||||
configuration, enforced by the coordinator.
|
||||
2. Every search records its model spend, task count and wall time. Spend is **reserved before each model call** (the
|
||||
call's maximum cost) and released after; wall time, memory, disk and retries are capped per task and in total; a
|
||||
budget is never exceeded, including by calls in flight.
|
||||
3. Paid runs need the maintainer's budget go-ahead with a ceiling, as all paid steps do.
|
||||
|
||||
### Requirement 5: Model access in the cluster
|
||||
|
||||
**User Story:** As the maintainer, I want agents in the cluster to reach a model without spreading keys.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. The agents' model key is a scoped, budget-limited key provided by the operator inside the cluster (AX v0.3.1
|
||||
injects only `gemini-api-secret` as `GEMINI_API_KEY`; the design names the exact mechanism agreed with the
|
||||
operator). Store credentials stay out of the cluster (presigned links, as today). For this arm, model calls inside
|
||||
the cluster supersede `unseen-repo-evaluation` Req 4.4 (model calls on the VM); that boundary is unchanged for all
|
||||
other arms. *(maintainer decision 2026-10-05: scoped key in the cluster)*
|
||||
2. The key never appears in a manifest, a log, a board entry or a record, and never shares a sandbox with repository
|
||||
code. *(Amended 2026-10-05, maintainer decision: trust domains are separated instead of isolated inside one task.)*
|
||||
The agent loop (the "brain", holding the key) runs on the VM or in its own AX task and executes no repository code;
|
||||
every tool call that touches the repository runs in a separate executor AX task without the key, exchanging
|
||||
commands and results through presigned store objects. A Firecracker executor runtime is considered only if the
|
||||
pilot's `could_not_build` rate shows gVisor cannot build the projects.
|
||||
|
||||
### Requirement 6: Feasibility pilot, false-proof qualification, then the unseen pool
|
||||
|
||||
**User Story:** As the maintainer, I want to know quickly whether search with proof works, and to admit BLOCK only on
|
||||
evidence that meets the 95% precision bar.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. **Feasibility pilot** (amended 2026-10-05 after review): about 20 known fix pairs from development corpora. It
|
||||
measures feasibility only and is reported as such: demonstrated rate on vulnerable sides, false proofs on fixed
|
||||
sides, `could_not_build` / `could_not_run` / `inconclusive` / `no_oracle` rates, spend, tasks and wall time. Its
|
||||
advantages (known location, labelled family, runnable-project bias) are stated in the record. Fixed-side searches
|
||||
never receive the vulnerable revision or its demo.
|
||||
2. Before any paid search, an infrastructure probe without model calls proves each owned oracle works inside a gVisor
|
||||
AX Task (database fixture for SQL, in-sandbox listener for SSRF, headless browser for XSS, file and process
|
||||
watches), that the artefact cannot reach the observers directly, and records image size, RAM and the full
|
||||
verification cost (up to 3 demos x 2 sides x 3 runs per search). Pilot exit: at least one family with at least 5 verified demonstrations on vulnerable sides and zero false proofs;
|
||||
otherwise the result is recorded and the arm is not scaled.
|
||||
3. **BLOCK admission per family** (deferred until the pilot passes; not part of the first tasks) requires a false-proof
|
||||
qualification. A **negative** is a search on code that should be safe: a fixed revision searched on its own (the
|
||||
search never sees the vulnerable revision), or an ordinary change. A **false proof** is the verifier observing the
|
||||
effect on that safe code, confirmed by adjudication (an ordinary-change alarm may be a real new bug and is
|
||||
adjudicated before it counts either way). Qualification needs zero false proofs over at least 73 negatives drawn
|
||||
from at least 73 distinct repositories, so the Wilson 95% upper bound on the false-proof rate per negative is at
|
||||
most 5%. This bounds the false-proof rate, not precision among flags; precision is reported separately from the
|
||||
pool. Repeated runs and repeated searches of one repository are one negative.
|
||||
4. **Coverage ceilings**: every run reports, separately, (a) the demonstrated ceiling: the share of vulnerable changes
|
||||
with a true candidate selected, an owned oracle, both sides runnable and a genuine differential, build and run
|
||||
failures in the denominator; and (b) the advisory ceiling: the share with a true candidate selected. Families
|
||||
without an owned oracle (access control, deserialisation, configuration and secrets) have a demonstrated and BLOCK
|
||||
ceiling of zero today; their advisory ceiling need not be zero.
|
||||
5. **G1 arm on the unseen pool**: scored with the pool's rules on **every visible finding** (BLOCK and advisory), with
|
||||
its family and location match; catch at the 1%, 5% and 10% false-alarm budgets; compared with the classifier arms;
|
||||
adopted only under the pool's no-regression rule on a fresh slice. No evaluation label, fixing revision or board
|
||||
from evaluation searches is ever an input to unseen search.
|
||||
6. Delayed results (a search finishing after the push) reach the user through the existing background delivery and
|
||||
are scored separately from results available at push time.
|
||||
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"feature_name": "search-with-proof",
|
||||
"created_at": "2026-10-05T12:00:00Z",
|
||||
"updated_at": "2026-10-07T13:30:00Z",
|
||||
"language": "en",
|
||||
"phase": "tasks-generated",
|
||||
"approvals": {
|
||||
"requirements": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"design": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"tasks": {
|
||||
"generated": true,
|
||||
"approved": false
|
||||
}
|
||||
},
|
||||
"ready_for_implementation": false,
|
||||
"pilot": {
|
||||
"status": "in_progress",
|
||||
"first_cluster_demonstration": "2026-10-07",
|
||||
"pair": "xiaomusic (path, prefix-bypass)",
|
||||
"false_proofs": 0,
|
||||
"exit_remaining": "path family >=5 demonstrations / 0 false proofs (Req 6.2), then >=73-negative BLOCK qualification (Req 6.3)"
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
# Implementation Plan
|
||||
|
||||
Scope: infrastructure probe and feasibility pilot only (Req 6.1-6.2). BLOCK qualification (6.3), the unseen-pool arm
|
||||
(6.5) and delivery (6.6) are deferred until the pilot passes. Implementation via Codex in a worktree; Claude reviews,
|
||||
gates, runs every networked or paid step.
|
||||
|
||||
- [ ] 1. Operator prerequisites (Claude, no code)
|
||||
- [ ] 1.1 Ask the operator: scoped spend-limited model key mechanism (reuse `gemini-api-secret` slot or a new
|
||||
injection), and whether a no-egress task class exists for verify runs. _Requirements: 5.1, 3.2_
|
||||
- [ ] 1.2 Record the answers in `kube-ax-cluster` memory and design.md; no key value anywhere. _Requirements: 5.2_
|
||||
|
||||
- [x] 2. Verifier and oracles (no model calls)
|
||||
- [x] 2.1 `src/openultrasast/search/verify.py`: fresh-sandbox runner per side; build, readiness check, read-only
|
||||
checkout and dependencies, untracked-file and runtime-patch refusal, 3 runs per side, consistency check,
|
||||
outcomes `demonstrated|inconclusive|could_not_build|could_not_run|no_oracle`. Artefact exit code ignored.
|
||||
_Requirements: 3.1-3.4_
|
||||
- [x] 2.2 Oracles owned by the verifier: SQL (database fixture + query log), path (file-access watch on a planted
|
||||
canary), XSS (headless browser execution check), SSRF (in-sandbox listener), command injection (process watch).
|
||||
Observers unreachable from the artefact. _Requirements: 3.1, 3.2, 3.5_
|
||||
- [x] 2.3 Unit tests with hand-written vulnerable/fixed toy apps per oracle, including gaming attempts (reading the
|
||||
canary, revision sniffing, patching a dependency, contacting the listener directly): each must not yield
|
||||
`demonstrated`. _Requirements: 3.2, 3.3_
|
||||
|
||||
- [ ] 3. Image and infrastructure probe (no model calls)
|
||||
- [ ] 3.1 `ousast-search-task` image from the AX task runner: Python, Node/TypeScript, PHP, Java toolchains, the
|
||||
oracle fixtures, headless browser; built by CI to GHCR, pinned by digest. _Requirements: 2.3_
|
||||
- [ ] 3.2 Probe: the toy apps from 2.3 verified as AX Tasks on kube-ax (gVisor) and on kind. Record image size,
|
||||
RAM peak, disk, wall time per verification and per 18-run search, oracle success per family, and that observers
|
||||
are unreachable. Record `benchmarks/measurements/2026-10-xx-search-probe/record.json`. _Requirements: 6.2_
|
||||
- [ ] 3.3 Gate: every oracle works under gVisor, or the family is marked `no_oracle` for the pilot.
|
||||
|
||||
- [x] 4. Board, coordinator, budgets
|
||||
- [x] 4.1 Board in the memory store as immutable versioned blobs (`plane/memory.py`); facts, intents, hints with
|
||||
writer step and task; coordinator is the only writer; tasks read and write through presigned links.
|
||||
_Requirements: 1.1-1.3_
|
||||
- [x] 4.2 Spend reservation before each model call in `plane/budget.py`, settled after; per-task memory, disk,
|
||||
cleanup caps; total wall-time and retry caps; tests that in-flight calls cannot overrun. _Requirements: 4.1, 4.2_
|
||||
- [x] 4.3 Dispatch reason, explore and verify as AX Tasks through `benchmarks/ax/batch.py` (extend its environment
|
||||
contract beyond numeric deadlines), VM lane as fallback, resume from checkpoints. _Requirements: 2.3, 2.4_
|
||||
|
||||
- [ ] 5. Agent worker
|
||||
- [x] 5.1 Reason step (no tools): board snapshot in, `complete` / up to 3 intents / nothing out. _Requirements: 2.1_
|
||||
- [x] 5.2 Explore step: tool loop (`list_files`, `read_file`, `grep`, `run`, `write_demo`, `finish`) via the
|
||||
existing OpenAI-compatible client; model client in a separate process and user from executed code; one fact
|
||||
or a recorded failure. _Requirements: 2.2, 5.2_
|
||||
- [ ] 5.3 Smoke test with a stub model (no spend) end to end on one toy pair on kube-ax.
|
||||
|
||||
- [ ] 6. Feasibility pilot (paid, after go-ahead)
|
||||
- [x] 6.1 Select 20 development-corpus pairs (never v3 or the unseen pool) in the four pool families with an owned
|
||||
oracle, across the five languages; record selection advantages. _Requirements: 6.1_
|
||||
- [ ] 6.2 Run 3 searches, measure real cost per search, ask the maintainer for the pilot ceiling. _Requirements: 4.3_
|
||||
- [ ] 6.3 Run the pilot: both sides per pair, fixed side blind to the vulnerable revision; record demonstrated
|
||||
rate, false proofs, failure-outcome rates, both coverage ceilings, spend, tasks, wall time.
|
||||
_Requirements: 6.1, 6.4_
|
||||
- [ ] 6.4 Exit decision against 6.2 and report to the maintainer; on pass, spec tasks for qualification and the
|
||||
unseen-pool arm. _Requirements: 6.2_
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Group 2 (2026-10-05): independently reviewed; `tests/test_search_verify.py` has 32 passing tests and one
|
||||
skipped SSRF integration test because this host forbids even localhost sockets. The namespace handoff and
|
||||
listener request parsing are tested separately; the full SSRF effect still needs the task 3.2 probe before
|
||||
claiming that oracle operational. The module audit and neighbouring sandbox/regression tests pass.
|
||||
- The local lane requires Linux Bubblewrap and util-linux, reuses `SandboxJob`/`SandboxResult`, and fails closed
|
||||
without isolation. Artefacts receive a batch CLI `TARGET` bridge and private scratch; HTTP apps need a trusted
|
||||
CLI adapter. Only trusted `Side.build_command` outputs can enter the app runtime (read-only `/build`);
|
||||
artefact `build.sh` writes scratch only. XSS requires a trusted browser executor, otherwise `no_oracle`.
|
||||
|
||||
- Tasks 3.1 (files only) and group 4 (2026-10-05): independent review approved; final offline validation:
|
||||
107 passed, 2 skipped (Chromium namespace/socket restrictions and the existing SSRF localhost restriction).
|
||||
Ruff and changed-source mypy pass; module audit: 160 load-bearing, 6 standalone, zero orphaned. No network
|
||||
actions, model calls, image builds, pushes or commits were performed. Task 3.1 remains unchecked until the
|
||||
image is built/published and a real digest is available; CI now includes `plane/Dockerfile.search-task`.
|
||||
- The coordinator accepts an injected executor. `BatchExecutor` uses the repository's `benchmarks.ax.batch`
|
||||
transport with one attempt per admission; coordinator owns the global retry budget and VM fallback. It requires
|
||||
a repository checkout import path and a trusted worker command; reason/explore workers remain group 5.
|
||||
Per-task memory, disk and cleanup ceilings are passed in the worker contract; worker enforcement and gVisor
|
||||
compatibility still require group 5 and the 3.2 probe. Only trusted verification results can meet the goal.
|
||||
- Board heads are immutable memory-store hashes; retain the returned head with the host run to resume. Unknown
|
||||
interrupted model calls conservatively consume their full reservation; settled failures with zero spend release
|
||||
it. Cleanup exceptions stop dispatch. Chromium uses a fresh profile in a private network namespace; injected
|
||||
script must call `alert("ousast-xss")`, while escaped/reflected payloads cannot produce the verifier DOM marker.
|
||||
|
||||
- Tasks 3.2 and 5.1–5.3 offline implementation (2026-10-05, explicitly authorized): packaged five-family
|
||||
toy pairs and real/two-gaming demos in `search/probe_apps/`; `python -m openultrasast.search.probe` reports
|
||||
per-side stage times, image versions, child/self peak RSS, retained scratch usage and sampled verification
|
||||
scratch peak. Generic isolation and later oracle namespace failures are instrument failures, never misses.
|
||||
Optional RESULT_URL uses an object-scoped urllib PUT. `benchmarks.ax.search_probe` reuses AX/Docker lanes
|
||||
(kind via its kubeconfig), with `ousast-engine-search-probe-` task names.
|
||||
- Worker reason/explore uses the existing OpenAI-compatible client with retries disabled and SpendBudget
|
||||
reservation before each call. Repository commands run only in the scrubbed sandbox with bounded tmpfs;
|
||||
the key-absence, timeout, output limit and disk limit tests execute real isolated processes. The scripted
|
||||
model/fake coordinator executor smoke reaches `demonstrated` through the real path oracle verifier.
|
||||
- Independent review APPROVED the requested offline scope. Fresh requested search/AX/plane tests plus module
|
||||
audit: 87 passed, 2 skipped, exit 0; focused new tests: 15 passed; Ruff and changed-source mypy pass.
|
||||
Audit: 162 load-bearing, 6 standalone, zero orphaned. Tasks 3.2 and 5.3 remain unchecked for their original
|
||||
live-cluster criteria; their code/local smoke is implemented. No network, paid calls, image builds, pushes
|
||||
or commits. Offline wheel build could not resolve the uncached uv-build backend, so installed-wheel and
|
||||
real AX/gVisor/kind/Docker image validation remain gaps.
|
||||
|
||||
- Task 6.1 and executor smoke (2026-10-05, explicitly authorized): added
|
||||
`benchmarks/search/executor_smoke.py` and `pilot_select.py`, with offline tests and usage documentation.
|
||||
The smoke drives one managed, key-free executor mailbox and records per-command timing/output measurements
|
||||
and sampled workspace peak; AX and Docker lifecycle cleanup is covered by fake-lane tests. No live task ran.
|
||||
Pilot selection read 10 development catalogs (873 rows, 904,050 bytes), retained 192 eligible pairs and selected
|
||||
20 distinct repositories: four per language and five per family. Identities live only in the gitignored local
|
||||
manifest; the aggregate record is `benchmarks/measurements/2026-10-05-search-pilot-selection/record.json`.
|
||||
All 20 selections have unknown manifest/test-suite evidence; no matching local cache or frozen unseen-pool
|
||||
metadata was present. These are feasibility inputs, not proof of buildability or qualification.
|
||||
Both task-local independent reviews approved after a reserved-cache-symlink regression was fixed.
|
||||
Fresh requested search/AX/plane tests, module audit, independent/reserved and unseen guards, and label guards:
|
||||
287 passed, four skipped, exit 0. Skips: Chromium isolation, two localhost-socket cases, and no frozen pool.
|
||||
Ruff and changed-source mypy pass; audit: 166 load-bearing, six standalone, zero orphaned.
|
||||
No network or commits. The requested main merge could not update read-only Git metadata; HEAD and main
|
||||
had identical file trees. Live AX/Docker executor smoke and oracle probes remain unmeasured.
|
||||
|
||||
- **FIRST CLUSTER DEMONSTRATION (2026-10-07) — pilot is live (6.2/6.3 in progress).** search-with-proof produced its
|
||||
first autonomous `demonstrated` finding end to end on kube-ax: pair 13 (hanxi/xiaomusic, path, prefix-check-bypass
|
||||
class), vulnerable side `demonstrated` (3/3 affected, 0/3 safe, $0.37), fixed side blind negative `inconclusive`
|
||||
with 0 false proofs ($0.56); pair total $1.22, image e2c8b460. Record:
|
||||
`benchmarks/measurements/2026-10-07-search-first-cluster-demonstration/`. This closes the handoff's open question.
|
||||
The infra probe (3.2) passed earlier; the pilot (6.x) has run many times. Enabling fix chain, each merged + paid-run
|
||||
validated: (1) parity (name the observable to the agent), (2) verify-time-only (observable injected at verify time,
|
||||
absent in exploration — ends the explore reachability trap), (3) fair-share pilot budget (no starvation across
|
||||
searches), (4) served-root guidance (demo MUST serve `${served_root}`), (5) oracle prefix-bypass coverage (sibling
|
||||
canary for `startswith`-without-separator path traversal), (6) control-signal feedback (served-root hint so the
|
||||
agent self-corrects wiring). Plus the keystone (positive control → `could_not_run`, never a silent clean-looking
|
||||
`inconclusive`). NEXT: fan out the path family to ≥5 demonstrations / 0 false proofs (6.2 exit), then the
|
||||
≥73-negative BLOCK qualification (6.3). glance (classic traversal) also demonstrates once its npm `filed`→`mime@4`
|
||||
build-break is pinned.
|
||||
@@ -1,52 +0,0 @@
|
||||
# Search-with-proof: where each stage runs (verification architecture)
|
||||
|
||||
Recorded 2026-10-07. search-with-proof has two stages across a trust boundary: a Cairn-inspired
|
||||
directed state-space **search**, then a DAST-style **proof (verification)**.
|
||||
|
||||
## The two stages
|
||||
|
||||
1. **Search (Cairn-inspired).** A blackboard of facts, intents and hints. A reasoner proposes
|
||||
independent, non-overlapping intents; tool-using explorers each confirm one fact against the
|
||||
code. The goal is a declarative demo (a candidate PoC), never proof by itself.
|
||||
2. **Proof (DAST verification).** The verifier builds and starts the app, drives the demo's steps
|
||||
through the app's real interface, and watches a verifier-owned canary for a **differential**
|
||||
between the vulnerable and fixed revisions: the owned effect appears on the vulnerable side and
|
||||
not on the fixed side (3/3 vs 0/3). Only this differential earns `demonstrated`.
|
||||
|
||||
## Trust boundary — what runs where
|
||||
|
||||
The brain runs on the VM; the agent's tool executions and the DAST proof run as isolated tasks in
|
||||
the kube-ax cluster. Secrets never enter the cluster.
|
||||
|
||||
| Component | Location | Notes |
|
||||
|-----------|----------|-------|
|
||||
| Coordinator, board, budgets | VM (host) | Orchestrates the search; owns the global budget. |
|
||||
| Reasoner + explorer **model calls** | VM (host) | The model key lives only on the VM. This is the brain. |
|
||||
| Explorer **tool runs** (`list_files`, `read_file`, `grep`, `run`, build) | kube-ax AX Task (gVisor) | Carry no secrets; inputs/results via presigned object links. |
|
||||
| **DAST proof / verification** | kube-ax AX Task (gVisor) | Builds, starts, attacks and observes the app. One task per revision. |
|
||||
|
||||
- **gVisor is the isolation boundary.** The proof runs untrusted exploit code against a running
|
||||
app, so it is sandboxed per-task in the cluster.
|
||||
- **Secrets stay on the VM.** Executor and verifier tasks carry no credentials; they exchange
|
||||
checkout archives, demos and results through presigned object-store links. Per-host egress is
|
||||
restricted (verify tasks reach the files host only).
|
||||
|
||||
## Verification lanes (the `--verify-lane` choice)
|
||||
|
||||
- **`ax` (default for paid pilot runs):** verification runs **inside kube-ax** as gVisor AX Tasks,
|
||||
one per side. This is how every paid pilot run this session executed the proof.
|
||||
- **`vm` (local):** verification runs in **Bubblewrap on the host**, no cluster. Used for the
|
||||
zero-cost ceiling proofs (a hand-authored demo verified locally with no model and no cluster),
|
||||
which is how glance and xiaomusic were confirmed demonstrable before any spend.
|
||||
- **`docker` (replay):** `benchmarks.search.demo_replay` runs a stored demo with no model calls in
|
||||
a local Docker lane.
|
||||
|
||||
## Why the proof is dynamic, not abstract interpretation
|
||||
|
||||
Abstract interpretation (the static engine and the decision model) is the cheap, generalizable
|
||||
**triage/ranker**: it selects and orders candidate regions across any repo without executing. It
|
||||
plateaued as a **blocking** gate because it could not prove the required precision. search-with-proof
|
||||
is the **dynamic** confirmation: a demonstrated finding is ground truth by execution, so it earns a
|
||||
block without calibration. The per-app harness cost (building and running the app, wiring its served
|
||||
directory or database to the oracle fixture) is the price of the dynamic proof and is why proof pays
|
||||
off where a runnable harness exists, with abstract interpretation remaining the static backbone.
|
||||
@@ -1,32 +0,0 @@
|
||||
# Brief: does the safety net work on repositories it has never seen?
|
||||
|
||||
Maintainer, 2026-10-04: "we want to spread the safety net across repos in general, unseen code. that was the purpose
|
||||
of the AB tests to see if our approach is generalizeable." And: "we should run our tests / runs in kube. now we can
|
||||
speed it up and design the workloads for ax directly. kind is only a fallback."
|
||||
|
||||
## Why this spec exists
|
||||
|
||||
The A/B experiments so far (exp-003, exp-004, exp-005) score the decision engine on repository-held-out folds of known
|
||||
fix pairs. That measures whether the vulnerable side of a pair scores above its fixed side. It does not measure what a
|
||||
safety net on an arbitrary repository lives on:
|
||||
|
||||
- on an ordinary push, does it stay quiet? (false-alarm rate)
|
||||
- on a push that introduces a real vulnerability, does it say so? (catch rate)
|
||||
|
||||
Populations v1 and v2 measured the engine scan on fresh cases once (1 of 11, 0 of 17) and are spent. Nothing since has
|
||||
measured push-level behaviour on unseen repositories.
|
||||
|
||||
## What this spec delivers
|
||||
|
||||
1. A frozen pool of repositories never used anywhere in development, with two kinds of change each: real
|
||||
vulnerability-introducing commits and ordinary commits.
|
||||
2. A push-level metric: each change replayed through the hook exactly as a user would run it, and through other
|
||||
arms (decision engine, engine evidence) where an experiment asks.
|
||||
3. Execution designed for the kube-ax cluster from the start: one push replay is one AX Task; kind and the VM are
|
||||
fallbacks.
|
||||
4. A baseline for today's hook, then a table of arms scored on the same pool.
|
||||
|
||||
## Not in scope
|
||||
|
||||
Changing detection. This spec only builds the measurement. Improvements are separate, registered experiments judged
|
||||
by it.
|
||||
@@ -1,518 +0,0 @@
|
||||
# Design Document
|
||||
|
||||
## Overview
|
||||
|
||||
This spec builds the instrument for roadmap G0: a frozen pool of repositories that no development step has seen,
|
||||
and a push-level score on it. The score is what a user of the hook sees. On a vulnerability-introducing push, did
|
||||
the hook say so (catch rate)? On an ordinary push, did it say anything (false-alarm rate)? Replays run as AX Tasks
|
||||
on kube-ax. kind and the VM run the same tasks as fallbacks.
|
||||
|
||||
Four facts from the code shape the design:
|
||||
|
||||
- **Populations already have a freeze-and-guard pattern.** `benchmarks/independent/freeze.py` verifies cases by
|
||||
reading git only, `evaluate.py` scores with fixed rules and Wilson intervals, and
|
||||
`tests/test_independent_population.py` fails when a reserved repository is named anywhere in `benchmarks/`,
|
||||
`src/` or `tests/` outside its directory. Harvest batch 3 used that guard as a black box: candidates written,
|
||||
guard run with output discarded, subsets bisected on red. That is how v3 stays unread.
|
||||
- **The hook can already replay a change.** `ousast pre-push PATH --base B --head H --artifact F` resolves two
|
||||
local commits (`snapshot.py`, `resolve_replay`: no fetch, no merge-base), runs quick rules and the engine inside
|
||||
one deadline (default 30 s, engine inline) and writes a JSON artifact. The terminal text has one `Skipped:` line
|
||||
per skipped check. The artifact holds three finding streams: admitted defects, advisory engine findings
|
||||
(new or worsened, vetoed only by capability) and quick-rule matches with their CWE.
|
||||
- **The AX pattern works but lives in one script.** `engine_trace_ax.py` (manifest, egress checks, presigned
|
||||
objects, retry once under a new name, cleanup) and the lane scheduler in `engine_trace.py` (`run_plan`: fixed
|
||||
lanes, fallback queue, `progress.json`, `STOP`, resume by input digest) are engine-specific.
|
||||
- **Placement on kube-ax is random.** One worker per node, shared by all actors. Two heavy tasks fit per worker,
|
||||
but only two concurrent tasks in all guarantee at most two on one worker.
|
||||
|
||||
### Goals
|
||||
|
||||
- A pool frozen by code against every used source (Req 1), with two kinds of change per repository (Req 2).
|
||||
- A scorer whose numbers mean what the hook shows, with silent zeros visible (Req 3).
|
||||
- One replay is one AX Task via a dispatcher shared with the engine (Req 4); a baseline and fresh slices (Req 5).
|
||||
|
||||
### Non-Goals
|
||||
|
||||
Detection changes, population v3, model calls in the cluster, languages the hook does not analyse.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
M[pool manifest slice] --> D[batch dispatcher on VM]
|
||||
D --> P[presign spec GET and record PUT]
|
||||
P --> T[AX Task ousast-engine-replay]
|
||||
T --> G[shallow fetch base and head from GitHub]
|
||||
G --> H[ousast pre-push --base --head]
|
||||
H --> R[record tar: artifact, text, instrument]
|
||||
R --> S[(store via presigned PUT)]
|
||||
D -->|sandbox died twice or OOM| V[VM docker lane]
|
||||
D -->|cluster down| K[kind lane, same tasks]
|
||||
V --> S
|
||||
K --> S
|
||||
S --> C[scorer: catch, false alarm, coverage]
|
||||
S --> A[model arms on VM, budgeted]
|
||||
A --> C
|
||||
C --> O[record: counts and digests]
|
||||
```
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### 1. Pool sourcing and eligibility (Req 1)
|
||||
|
||||
**Decision 1.1: sources and ecosystems.** Candidates come from GitHub reviewed advisories (`gh api /advisories`,
|
||||
the enumeration harvest-3 used) for pip, npm, maven and composer, published 2019 or later, whose CWE maps to one of
|
||||
the seven families through the project's family CWE table. Each needs a linked fix commit with one parent in one
|
||||
repository. OSV records for the same advisory give the first affected version. That version is a cross-check for
|
||||
the introducing commit (section 2). Reason: these four ecosystems are the five languages the hook analyses
|
||||
(Python, JavaScript, TypeScript, PHP, Java). GHSA links commits and CWEs; OSV adds version brackets. Rejected:
|
||||
drawing from every ecosystem (Go, Ruby, C#). That would mostly measure missing frontends, which is already known.
|
||||
Coverage still reports uncovered files inside the drawn repositories. Rejected: NVD and published fix datasets.
|
||||
NVD links commits weakly, and the datasets are mined corpora that development or model training has likely seen.
|
||||
|
||||
**Decision 1.2: eligibility by code against every used source.** `benchmarks/unseen/eligibility.py` builds the
|
||||
used set before the draw. It never prints names; it records counts per source and a digest of the set.
|
||||
|
||||
| Source (Req 1.1) | Where it is read |
|
||||
| --- | --- |
|
||||
| Pair corpora | every catalog, recipe and dataset file under `benchmarks/pairs/`, plus the local pointer-pair manifests |
|
||||
| Populations v1, v2 | `population-v1.toml`, `population-v2.toml`, read directly |
|
||||
| Population v3 | never opened; the existing guard test only (below) |
|
||||
| Harvest records | the batch catalogs under `benchmarks/pairs/advisory-fixes*` (the records hold counts only) |
|
||||
| Memory examples | `example` rows of the local FileStore and the S3 store (`repo` field) |
|
||||
| Experiment units | `plane/experiments/*.units.jsonl` and registered units files |
|
||||
| Measurement records | everything under `benchmarks/measurements/` and `benchmarks/experiments/` |
|
||||
| Anything else | a text sweep for `github.com/<owner>/<name>` over `benchmarks/`, `src/`, `tests/`, `plane/`, `.kiro/` |
|
||||
|
||||
Names are normalised (lower case, no `.git`, no trailing slash). Each candidate is resolved through the GitHub API
|
||||
to its canonical `full_name`, its former names (redirects) and its fork network (`parent`, `source`). A candidate
|
||||
is out if any of these is in the used set. A fork shares code with what it was forked from.
|
||||
|
||||
v3 is checked like harvest-3 did. The draft manifest is written into `benchmarks/unseen/`. Then
|
||||
`pytest -q tests/test_independent_population.py -k referenced` runs with its output discarded. The guard already
|
||||
sweeps `benchmarks/` outside the population directory. On red, the candidate list is bisected by rewriting the
|
||||
draft until green, and the dropped count is recorded. No v3 name is ever printed or read by a person.
|
||||
|
||||
**Decision 1.3: post-cutoff preference.** An advisory published after 2026-06-01 (harvest-3's cutoff for the
|
||||
decision engine's model) is flagged `post_cutoff`. Within each ecosystem stratum post-cutoff advisories are drawn
|
||||
first. Results are reported for both groups. Reason: a post-cutoff advisory cannot be in the model's training
|
||||
data as a known vulnerability. Rejected: post-cutoff only. Harvest-3 kept 12 post-cutoff pairs out of 251, and
|
||||
three slices of 100 repositories cannot be filled from that window.
|
||||
|
||||
**Decision 1.4: licences.** The licence is read at the pinned head commit (LICENSE file, then the API's SPDX id).
|
||||
Only permissive licences are published by name: MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, Unlicense,
|
||||
Zlib, PostgreSQL, BSL-1.0. Everything else (copyleft, source-available, none) goes to the gitignored
|
||||
`benchmarks/unseen/private/pool-p1.toml`. The committed manifest then holds an opaque id and the sha256 of
|
||||
(URL, commits) for that entry, so the freeze digest still covers it. Reason: Req 1.4 read literally. Rejected:
|
||||
"OSI-approved" as the bar, as population v2 used. It is wider than the requirement says (open question 1,
|
||||
resolved 2026-10-04: copyleft stays private).
|
||||
|
||||
Also required: anonymous HTTPS, at most 500 MB, at least 40 eligible ordinary commits, one per fork network.
|
||||
|
||||
### 2. Change extraction (Req 2.1, 2.2)
|
||||
|
||||
Extraction runs on the VM over a full `--filter=blob:none` clone in a scratch cache. It uses git only, like
|
||||
`freeze.py`. No scanner, engine or model touches a pool repository before the freeze.
|
||||
|
||||
**Decision 2.1: the vulnerability-introducing range.** Let F be the fix and P its parent. The vulnerable lines are
|
||||
the old-side lines of F's hunks in the vulnerable function. For a pure-addition fix (an absence bug such as a
|
||||
missing guard), they are the vulnerable function's own lines.
|
||||
|
||||
1. **`introducing` (preferred).** `git blame -w -M -C --porcelain P -L <lines> -- <file>`. The candidate V is the
|
||||
commit that last wrote the sink line (for absence bugs, the function's declaration). V is accepted if all hold:
|
||||
at V's first parent the sink line's normalised text is absent from the file's vulnerable function; V is not a
|
||||
merge or root commit; V changes at most 50 files and 2,000 lines; when OSV names a first affected version, V is
|
||||
reachable from that tag and not from the previous release tag. If the text is present at V's parent, blame
|
||||
walks back from there, at most five steps. The replay is `base = V^`, `head = V`.
|
||||
2. **`last_touch` (fallback).** When no V passes, `git log -L :<function>:<file> P` gives the newest commit L
|
||||
before F that changed the vulnerable function. The replay is `base = L^`, `head = P`. The vulnerable code is in
|
||||
head, and the range contains a change to the vulnerable function.
|
||||
|
||||
Each change records its method, family, vulnerable function (resolved in head), head-side changed lines,
|
||||
`post_cutoff` and the advisory id.
|
||||
|
||||
Bias of each, stated in the record. `introducing` inherits SZZ noise: blame finds the last writer, not the author
|
||||
of the flaw. A refactor, a move the heuristics miss or a reformat can stand in for the real introduction. The
|
||||
absence check, the walk-back and the version bracket cut this down, but do not remove it. The bias runs both
|
||||
ways: a wrong V may introduce nothing (a miss is not the hook's fault) or may move a flaw (the delta-based hook is
|
||||
right to call it existing). `last_touch` is pessimistic for a delta hook. The flaw is often already in base, so
|
||||
the hook's novelty rule correctly stays quiet and the change counts as a miss. Catch is therefore reported per
|
||||
method. The headline is `introducing`; `last_touch` is a lower bound. Rejected: fix-parent against a fixed old
|
||||
base (for example 50 commits back). The range often misses the vulnerable function entirely and grows without
|
||||
bound on busy repositories.
|
||||
|
||||
Label check: every third `introducing` change is read by an agent before the freeze, with no model call, as
|
||||
harvest-3 reviewed diffs. It is marked `confirmed`, `refactor` or `unclear`. The confirmed share is the label
|
||||
noise estimate. The scorer also reports catch on the confirmed subset.
|
||||
|
||||
**Decision 2.2: ordinary commits.** The candidates are non-merge commits on the first-parent line of the default
|
||||
branch that change at least one source file of an analysed language. Excluded:
|
||||
|
||||
- explicit security signals in the message: `secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|exploit|sanitiz|escap`;
|
||||
- reverts (`^Revert`, `This reverts commit`);
|
||||
- dependency bumps (bot authors such as dependabot and renovate, `bump|upgrade .* from`, or only manifest and lock
|
||||
files changed);
|
||||
- commits whose source changes are all generated or vendored (`vendor/`, `node_modules/`, `dist/`, `*.min.js`,
|
||||
lock files, `linguist-generated` in `.gitattributes`, "generated" headers);
|
||||
- any fix commit of any advisory of the repository, and any commit touching a known vulnerable function of it;
|
||||
- commits over 1,000 changed source lines (counted, reported).
|
||||
|
||||
Twelve are drawn per repository with a seeded random draw. Strata are three time thirds of the repository's
|
||||
history crossed with four size buckets of changed source lines (1-10, 11-50, 51-200, 201-1,000). Allocation is
|
||||
proportional to the repository's own commits per cell, with at least one per time third. Each change records base
|
||||
(its first parent), head, changed files and line ranges. Reason: proportional allocation keeps the false-alarm
|
||||
rate a per-push rate as a user meets it. Rejected: `freeze.py`'s broad `SECURITY_WORDS`. It also drops commits
|
||||
saying "query", "token", "session" or "valid", which removes the ordinary commits where false alarms are most
|
||||
likely, and the rate would look better than it is. Rejected: docs-only commits. They are trivially quiet and
|
||||
would dilute the rate.
|
||||
|
||||
Label noise: an ordinary commit may itself introduce an unknown vulnerability, so a flag counted as a false alarm
|
||||
may be true. The false-alarm rate stays as Req 3.2 defines it. Flagged ordinary changes are adjudicated beside it
|
||||
(all if 30 or fewer, else every third) against protocol-v2's standard, and the adjudicated share is reported.
|
||||
|
||||
### 3. Sizing and slices (Req 2.3, 5.2)
|
||||
|
||||
Wilson 95% interval, z = 1.96: centre (p + z²/2n)/(1 + z²/n), half width z·sqrt(p(1-p)/n + z²/4n²)/(1 + z²/n).
|
||||
Taking the larger distance from p as the width, the smallest n that meets the bar, computed exactly:
|
||||
|
||||
| Rate | Assumed p | Bar | n needed |
|
||||
| --- | --- | --- | --- |
|
||||
| false alarm | 0.05 | +/- 3 points | 315 |
|
||||
| false alarm | 0.10 | +/- 3 points | 483 |
|
||||
| false alarm | 0.20 | +/- 3 points | 756 |
|
||||
| catch | 0.10 | +/- 10 points | 62 |
|
||||
| catch | 0.25 | +/- 10 points | 88 |
|
||||
| catch | 0.40 | +/- 10 points | 97 |
|
||||
|
||||
Catch: 100 vulnerability-introducing changes per slice, one per repository. At p = 0.4, Wilson gives
|
||||
[0.309, 0.498], within 10 points. One per repository keeps them independent.
|
||||
|
||||
False alarm: ordinary changes of one repository are correlated (noisy code flags often). Wilson assumes
|
||||
independence, so the count is inflated by a design effect 1 + (m - 1)·ICC. With m = 12 per repository and an
|
||||
assumed ICC of 0.05, that is 1.55. 100 repositories × 12 = 1,200 changes, effective n 774 ≥ 756. The plain Wilson
|
||||
interval at p = 0.2, n = 1,200 is about +/- 2.3 points. A repository-cluster bootstrap interval is reported beside
|
||||
it, and the ICC measured on slice 1 is recorded. If it exceeds 0.05, the next pool sizes up. A frozen slice is
|
||||
never changed.
|
||||
|
||||
**Decision 3.1: slices.** Pool p1 has three slices of 100 repositories each (100 + 1,200 changes per slice; 300
|
||||
repositories and 3,900 changes in all). Repositories are assigned by a seeded shuffle, stratified by ecosystem and
|
||||
`post_cutoff`. Slice 1 serves the baseline and the G1 arms table. Slices 2 and 3 serve the first two adoption
|
||||
decisions. When they are spent, pool p2 is drawn by the same code, and it excludes p1. Reason: each slice alone
|
||||
meets the Req 2.3 widths, so any decision can stand on one slice. Rejected: one large pool scored whole. Every
|
||||
decision would then see every repository, which Req 5.2 forbids. Rejected: ten small slices. None would meet the
|
||||
widths. Per family, 100 changes split over seven families give intervals of about +/- 20 to 25 points. They are
|
||||
reported, not sized (open question 3).
|
||||
|
||||
Use ledger: `benchmarks/unseen/usage-p1.jsonl` is append-only, one row per use (slice, decision or experiment id,
|
||||
date, result digest). A slice with a row whose purpose is `informed` cannot appear again as `qualifies` for that
|
||||
change. A test checks this.
|
||||
|
||||
### 4. Scoring rules (Req 3.1, 3.2, 3.3)
|
||||
|
||||
Each replay uses the hook's defaults: `--deadline 30`, `--engine inline`, `--max-regions 500`, `--mode advisory`.
|
||||
That is what a user's push gets. The text output is kept with every `Skipped:` line, and so is the artifact.
|
||||
|
||||
**Decision 4.0: a registered long-deadline arm** (maintainer decision 2026-10-04, open question 5). Beside the
|
||||
default replay, slice 1 is replayed once more with `--deadline 300` and every other flag unchanged. The arm is
|
||||
registered (manifest committed, `unseen: {pool: p1, slice: 1}`) before its first replay. G1 then reports both
|
||||
what the user sees (30 s) and what the engine could catch given time (300 s), so a miss caused by the deadline is
|
||||
told apart from a miss by the engine. It is a measurement arm, not a proposed default: adopting a longer hook
|
||||
deadline would be a separate decision under 7.2.
|
||||
|
||||
**Decision 4.0a: calibrated cluster deadline and native check** (maintainer, 2026-10-05).
|
||||
The gVisor proof exhausted 30 seconds in snapshot preparation on a 1,725-file repository.
|
||||
Cluster replays therefore scale the 30/300-second arms by the median paired cluster/native
|
||||
time through the quick tier, rounded to integer seconds. Native runc replays retain 30/300 seconds.
|
||||
A deterministic sample (default 20, or all of a smaller proof manifest) runs both lanes at
|
||||
600 seconds for calibration. Report per-change stage times and median slowdown with bootstrap
|
||||
95% intervals for time through quick, engine build and engine queries separately. Incomplete
|
||||
timings or degraded scans invalidate calibration. Records contain counts, times and opaque digests.
|
||||
|
||||
A deterministic native sample accompanies the cluster slice. Compare whether each push flags
|
||||
anything, with a Wilson 95% interval. If its lower bound is below 95%, or any sample pair is
|
||||
missing/failed, flag the run and exclude cluster results from the baseline. Disagreements name
|
||||
change IDs only. The scorer reports applied deadlines and factors. Docker pulls explicitly after
|
||||
checking its data-root free space against an unpacked image size upper bound plus 1 GiB; failures
|
||||
retain sanitized stderr tails and exit codes. See the unseen README for coordinator commands.
|
||||
|
||||
**Decision 4.1: catch, per family.** A finding counts if its family is the change's family. Engine streams carry a
|
||||
family; a quick-rule match maps its CWE through the family CWE table. Its head location must also be in the
|
||||
vulnerable function or within 10 lines of the change's head-side hunks (protocol v1's window).
|
||||
|
||||
| Family | Location rule |
|
||||
| --- | --- |
|
||||
| injection, path, deserialization, output_encoding, untrusted_destination | sink line in the vulnerable function, or the changed lines +/- 10 |
|
||||
| access_control | the vulnerable function itself (the handler that lacks the check), or the changed lines +/- 10 |
|
||||
| config_secrets | the changed file and lines +/- 10 (file scope; function `<global>`) |
|
||||
|
||||
All three streams count: admitted defects, advisory engine findings and quick-rule matches. Each catch also names
|
||||
the stream that made it. Reason: the hook shows all three to the user. Rejected: admitted defects only. That
|
||||
scores the blocking path, which G3 decides separately, and would hide what users read.
|
||||
|
||||
**Decision 4.2: false alarm.** An ordinary change is a false alarm when any of the three streams holds at least one
|
||||
finding, of any family, anywhere. The alerts-only rate (admitted defects) is reported beside it.
|
||||
|
||||
**Decision 4.3: coverage, so zeros cannot hide.** Every change gets exactly one state:
|
||||
|
||||
- `instrument_failure`: no record, a failed fetch, the head commit not verified, or zero bytes of the changed files
|
||||
read. It is re-run once, then reported. It never counts as quiet or missed.
|
||||
- `unanalysable`: no changed file is in a language the hook covers (the hook's `language_not_covered` notice).
|
||||
- `quick_only`: the quick tier read the changed files, but no engine question completed on them.
|
||||
- `engine_covered`: at least one engine question completed on a changed file.
|
||||
|
||||
The task entry proves its input as AGENTS.md requires. It records files and bytes in the checkout, bytes of each
|
||||
changed file, the JVM's wall time and the hook's exit code. A replay faster than a JVM can start is flagged
|
||||
`suspect_fast` and goes to the failure path. Catch and false alarm are reported over all changes and over each
|
||||
coverage state.
|
||||
|
||||
### 5. The AX workload (Req 4)
|
||||
|
||||
**Decision 5.1: the image.** `plane/Dockerfile.engine-task` gains a second stage, `FROM engine AS replay`, built
|
||||
with `--target replay` and pinned by digest. It adds `git`, the package with its runtime dependencies and
|
||||
`benchmarks/unseen/replay_entry.py`. Joern, php-cli, Java 21 and the frontend-retention patch come from the engine
|
||||
stage, so both lanes run the same Joern. The import smoke runs `ousast pre-push --help`. The root image stays
|
||||
`ghcr.io/norandom/ax-task-runner:v0.3.1`. Rejected: a separate Dockerfile. It would duplicate the Joern install,
|
||||
and two Joern copies are two instruments that drift. Rejected: one stage for both. The engine image's isolated
|
||||
venv without yaml and boto3 is proven, and replay dependencies would change its digest on every bump.
|
||||
|
||||
**Decision 5.2: the task entry.** `replay_entry.py` is standard library only, like `engine_task_entry.py`, and
|
||||
reuses its `transfer` (egress readiness wait, three attempts, no URL text in errors).
|
||||
|
||||
1. GET `SPEC_URL` (presigned, `.json`): repository URL, base, head, change id, hook flags.
|
||||
2. `git init`, then `git fetch --depth 1 origin <head>` and `git fetch --depth 1 origin <base>` over HTTPS
|
||||
(GitHub serves commits by SHA). Verify both with `git cat-file -e`. The fetch is bounded at 180 s; the 500 MB
|
||||
repository cap bounds its size.
|
||||
3. Run `ousast pre-push <checkout> --base <base> --head <head> --artifact /tmp/out/push.json` with defaults, the
|
||||
backend heap at 1700 MB and the cluster JVM flags of `engine_trace_worker.py`. Capture stdout.
|
||||
4. Pack `push.json`, `push.txt`, `engine.json` (the head and base scan records and witness rows from the artifact,
|
||||
keyed by repository and revision, for engine-evidence arms) and `instrument.json` (section 4.3, image digest,
|
||||
step timings). PUT to `RESULT_URL` as `.dat`. The PUT is completion.
|
||||
|
||||
On any error it PUTs a failure record naming the step and HTTP status, as the engine entry does. Task names are
|
||||
`ousast-engine-replay-<slug>-<digest>`, the prefix the operator's egress watcher matches. No key or store
|
||||
credential enters the manifest. Only the two presigned URLs do, and the spec object keeps repository names out of
|
||||
AX status and cluster telemetry.
|
||||
|
||||
**Decision 5.3: the generic dispatcher.** `benchmarks/ax/batch.py` takes over from `engine_trace_ax.AX` and
|
||||
`engine_trace.run_plan`:
|
||||
|
||||
- `Item`: id, input objects (name to bytes), the result object name, command, extra env, input digest.
|
||||
- `Workload`: image digest, allowed URL env names, deadline, how to validate a received result.
|
||||
- Lanes: `ax` (manifest, apply, poll the result object and AX phase, delete, clean objects), `kind` (the same
|
||||
manifest against the kind AX), `docker` (the VM image, one container at a time).
|
||||
- Scheduler: two AX lanes plus one VM lane by default. A sandbox death is retried once under a new name; a second
|
||||
death or a JVM OOM goes to the VM fallback queue, which beats overflow on the VM lane. `progress.json` is written
|
||||
only by the scheduler. A `STOP` file halts new launches. Resume skips items whose stored record has the same
|
||||
input digest. Cleanup failure stops dispatching.
|
||||
|
||||
The engine pass becomes a workload over it, with the same tests. Reason: the egress, secrecy and cleanup logic took
|
||||
several fixes and must exist once. Rejected: a replay-specific copy of `engine_trace_ax.py`. Rejected: putting it in
|
||||
`src/` now. G4's `ousast plane scan` may need it there, and it moves when G4 is planned.
|
||||
|
||||
Concurrency is two AX tasks. Placement is random, and only two guarantees two per worker at most. Four would need
|
||||
the operator to add placement (open question 2, resolved 2026-10-04: stays at two, no placement change
|
||||
requested). Estimated wall time per slice: about 75 s per replay (fetch, 30 s hook, setup), 1,300 replays over
|
||||
three lanes, about 9 hours. The long-deadline arm (4.0) is bounded at about 345 s per replay; at that bound 1,300
|
||||
replays over three lanes take about 42 hours, less where the hook finishes early. It runs with `STOP` and resume
|
||||
across sessions.
|
||||
|
||||
**Decision 5.4: model arms on the VM.** Arms that need DeepSeek (the decision engine, the combiner) read the stored
|
||||
records on the VM. They build candidates from the head side's changed functions, score them, and take the change
|
||||
score as the maximum. Each runs under its experiment manifest's `budget_usd` through the metered client, dry run
|
||||
first, with responses cached. Nothing in the cluster calls a model.
|
||||
|
||||
**kind (Req 4.5)** runs the identical manifest and image through the `kind` lane. Every new task is proven there
|
||||
before kube-ax, as `plane-on-kubernetes` requires.
|
||||
|
||||
### 6. Records and the guard (Req 1.2, 1.3, 5.1)
|
||||
|
||||
`benchmarks/unseen/` holds `pool-p1.toml` (repositories, licences, slices, changes with base, head, method,
|
||||
family, function, lines), `freeze-p1.json` (used-set digests and counts per source, guard result, label-check
|
||||
counts, freeze digest = sha256 of the canonical manifest), `protocol-p1.md` (the scoring rules of section 4,
|
||||
committed before the first replay) and `usage-p1.jsonl`. The `private/` directory is gitignored.
|
||||
|
||||
The guard grows by one parametrised reservation. The existing sweep becomes a list of (reservation directory,
|
||||
names): `benchmarks/independent/` for populations and `benchmarks/unseen/` for pools, including the private
|
||||
manifest when present. It also sweeps `plane/`. A second test asserts that pool and population names are disjoint,
|
||||
and prints only a count on failure. A third asserts that a frozen pool's digest matches `freeze-p1.json`, so the
|
||||
pool is never edited. Raw per-change records stay outside the repository (`~/ousast-results/unseen/` and the
|
||||
store). Committed result records under `benchmarks/measurements/` carry counts, rates, intervals and digests,
|
||||
never names.
|
||||
|
||||
### 7. Metrics and reporting (Req 3.3, 3.4, 5.3)
|
||||
|
||||
`benchmarks/unseen/score.py` reports per family and overall: catch and false-alarm rate with Wilson 95%
|
||||
intervals, the cluster bootstrap interval for false alarms, coverage counts per state, catch per method,
|
||||
post-cutoff and per stream, and the alerts-only rate.
|
||||
|
||||
**Decision 7.1: arms at fixed budgets.** An arm with a score is thresholded per budget (1%, 5%, 10%). The
|
||||
threshold is the lowest score whose false-alarm rate on the slice's ordinary changes stays within the budget.
|
||||
Catch at that threshold is reported with its Wilson interval and the realised false-alarm rate. That in-slice
|
||||
threshold is labelled exploratory. For a qualification, the threshold is fixed beforehand on an earlier slice or
|
||||
on held-out folds, and it is applied unchanged. A binary arm (today's hook) has one operating point, reported with
|
||||
the budgets it meets. Rejected: AUC over changes (it averages operating points no user runs).
|
||||
|
||||
**Decision 7.2: next to held-out folds.** Experiment manifests registered after this spec lands carry
|
||||
`unseen: {pool, slice}`. `analyse` writes an `unseen` block into `result.json` next to the fold estimates: catch at
|
||||
each budget for each arm, and the paired difference B - A with an exact McNemar test on the 100 changes. The
|
||||
fold rule decides as today. The slice is recorded in the usage ledger.
|
||||
|
||||
**Decision 7.3: adoption gate on the unseen slice** (amended 2026-10-04, maintainer decision; Req 5.4; open
|
||||
question 4 resolved). Reporting is not enough. An arm B is adopted over the production arm A only if, on the
|
||||
untouched slice that qualifies it, both hold:
|
||||
|
||||
- catch at the 5% budget, with B's threshold fixed beforehand (7.1), is not significantly lower than A's: exact
|
||||
two-sided McNemar on the paired vulnerability-introducing changes, alpha 0.05;
|
||||
- for an arm with one operating point (a binary arm such as a hook setting), its false-alarm rate is not
|
||||
significantly higher than A's: exact two-sided McNemar on the paired ordinary changes, alpha 0.05, with the
|
||||
repository-cluster bootstrap interval of the difference reported beside it, since those changes are clustered.
|
||||
|
||||
`analyse` writes `unseen.gate: pass | regression | not_run` into `result.json`. `not_run` (no slice, slice
|
||||
already `informed` for this change, or more than 5% `instrument_failure`) blocks adoption like `regression`. The
|
||||
fold rule and this gate must both pass.
|
||||
|
||||
### 8. Increments
|
||||
|
||||
1. **Used set and guard.** `eligibility.py`, the extended guard and the usage-ledger test. Exit: guard green on an
|
||||
empty pool; a planted used repository, a renamed one and a fork are each rejected in tests.
|
||||
2. **Sourcing and extraction.** Candidates, both change kinds and the label check, on the VM. Exit: a draft for 300
|
||||
repositories with method counts, the confirmed share and exclusion counts.
|
||||
3. **Freeze p1.** Guard bisection for v3, then manifest, freeze record and protocol committed. Exit: freeze digest
|
||||
committed and guard green.
|
||||
4. **Dispatcher.** `benchmarks/ax/batch.py`, with the engine pass moved onto it. Exit: existing tests green and a
|
||||
two-pin engine smoke on kube-ax matching its stored records.
|
||||
5. **Replay task.** Image stage and entry, proven on kind, then kube-ax on five changes from this repository's own
|
||||
history (never pool changes). Exit: instrument fields show bytes read and realistic JVM time, and quick-tier
|
||||
output matches the VM lane.
|
||||
6. **Scorer.** `score.py` on synthetic records, including every coverage state and an instrument failure. Exit:
|
||||
unit tests green.
|
||||
7. **Baseline.** Today's hook, from a frozen export of HEAD, on slice 1 (1,300 replays). Exit: the record committed
|
||||
(counts and digests only) and a `baseline` row in the usage ledger. That is G0's exit.
|
||||
8. **Arms on slice 1 (G1)** (added 2026-10-04 with decisions 4.0 and 7.3). The registered long-deadline arm,
|
||||
the engine-evidence arm from the stored records, the model arms on the VM (5.4) under their budgets, and the
|
||||
`unseen` block with the adoption gate in `analyse`. Exit: one committed table of arms on slice 1 at the 1/5/10%
|
||||
budgets (counts, rates, intervals, digests), and `exploratory` rows in the usage ledger.
|
||||
|
||||
## Error Handling
|
||||
|
||||
A failed fetch, a missing commit or a hook crash gives a failure record, never an empty success. A dead sandbox is
|
||||
retried once, then the VM runs it. A store or AX cleanup error stops the dispatcher. A slice with more than 5%
|
||||
`instrument_failure` after re-runs is not scored; the cause is fixed first.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
Unit tests: used-set normalisation (case, `.git`, renames, forks), blame acceptance on a synthetic repository,
|
||||
ordinary-commit exclusions, Wilson values against section 3, budget thresholds, coverage states, the task manifest
|
||||
and dispatcher resume and fallback with a fake lane. Guard tests run in CI; cluster runs stay manual.
|
||||
|
||||
## Requirement Traceability
|
||||
|
||||
| Requirement | Design |
|
||||
| --- | --- |
|
||||
| 1.1 eligibility against every source, v3 via the guard | 1.2 |
|
||||
| 1.2 frozen manifest, never edited | 6, increment 3 |
|
||||
| 1.3 pool directory, guard extended | 6 |
|
||||
| 1.4 licences, private manifest | 1.4 |
|
||||
| 2.1 introducing changes, family, function, post-cutoff | 1.1, 1.3, 2.1 |
|
||||
| 2.2 ordinary changes across time and size | 2.2 |
|
||||
| 2.3 sizes for interval widths | 3 |
|
||||
| 3.1 default replay, skipped checks kept | 4, 5.2 |
|
||||
| 3.2 catch and false-alarm rules | 4.1, 4.2 |
|
||||
| 3.3 per family, Wilson, coverage | 4.3, 7 |
|
||||
| 3.4 arms at 1/5/10% budgets | 4.0, 7.1, increment 8 |
|
||||
| 4.1 one change one Task, prefix, image by digest | 5.1, 5.2 |
|
||||
| 4.2 HTTPS fetch, presigned record, no keys | 5.2 |
|
||||
| 4.3 generic dispatcher, concurrency, retry, fallback, resume | 5.3 |
|
||||
| 4.4 model arms on the VM within budgets | 5.4 |
|
||||
| 4.5 kind runs the same tasks | 5.3, 5 (kind paragraph) |
|
||||
| 5.1 baseline on slice 1, counts and digests | 6, increment 7 |
|
||||
| 5.2 fresh slice per decision | 3.1, usage ledger |
|
||||
| 5.3 experiments report unseen next to folds | 7.2 |
|
||||
| 5.4 adoption needs no significant unseen regression | 7.3 |
|
||||
|
||||
## Amendments (2026-10-11)
|
||||
|
||||
Folded in after the G0 pool was frozen; the original sections above stay as the design of record, these refine it
|
||||
where reality differed.
|
||||
|
||||
- **Persistence is bucket-first, not local disk.** The draw streams every stage to the `ousast-corpus` RustFS
|
||||
bucket and resumes from it, so a restart (even after a code change) is cheap. Layout: `unseen/<seed>/decisions/`
|
||||
(accept/reject per repo, key `sha256(repo+fix)`, idempotent), `unseen/<seed>/resolved/` (per-advisory
|
||||
resolution), `unseen/<seed>/advisories/list.json`, `unseen/repos/` (get_repo cache, **not** seed-scoped, shared
|
||||
across seeds), and `unseen/<seed>/pool/` (the frozen pool + provenance — the "don't redo sourcing" archive). The
|
||||
local journal is a within-process cursor only. Implementation: `benchmarks/unseen/bank.py` (`CorpusBank`). The
|
||||
VM's local disk is the scarce resource; run artifacts and replay outputs belong on RustFS (see steering
|
||||
`storage-streaming.md`). Credentials: `.env` `CORPUS_S3_*`; boto3 needs path-style + SigV4.
|
||||
- **Harvest cost is GitHub-rate-limit bound, measured yield ~2.2%.** Through the strict change criteria (1 vuln +
|
||||
12 stratified ordinary, >=40 eligible commits) ~2.2% of candidates are accepted, so "100 repos" is ~4,500
|
||||
candidate attempts at ~4 GitHub calls each — the bottleneck is the 5,000/hr API limit, not CPU or cores. Pure
|
||||
GitHub-API harvesting cannot replace the clone: the positive is the SZZ introducing commit and the negatives are
|
||||
stratified over full history, both needing repo history. The convergence fix was a full-objects clone (dropping
|
||||
`--filter=blob:none`, which caused ~600 per-blob lazy fetches) plus the bucket caches above.
|
||||
- **Record 07 is a machine-label baseline.** The pool's 100 vulnerability labels are SZZ-derived and unchecked
|
||||
(`freeze-p1.json` `label_check_counts`: 100 unchecked). The G0 baseline is reported as a machine-label number
|
||||
with that caveat; a human label pass is a later quality refinement, required before any published precision
|
||||
claim, not before record 07.
|
||||
- **Replay image provenance is pinned.** CI `engine-image.yml` builds `ghcr.io/norandom/ousast-replay-task` (the
|
||||
replay harness, which `COPY`s `/app/benchmarks/unseen/replay_entry.py`) and `ousast-engine-task` (the Joern
|
||||
hook), publicly pullable by digest. `ousast-runner` (the 248 MB ax-task-runner, no `/app/benchmarks`, no Joern)
|
||||
is **not** the replay image. The AX lane's store check (`benchmarks/ax/batch.py`) requires a public-HTTPS DNS
|
||||
presign-capable endpoint; how that and the cluster's store access are provided is operator-owned infra, not
|
||||
assumed here.
|
||||
|
||||
- **AX lane blocker (2026-10-11) — kube is the target lane, so this is tracked, not dismissed.** The replay
|
||||
*dispatches* correctly but tasks do not *complete*: `pins_done` stayed 0 for both the 5-change proof and the
|
||||
1,300-change slice 1. Sub-issues resolved along the way: (a) the replay image is `ousast-replay-task` (public);
|
||||
(b) the ax CLI must be `~/go/bin/ax` — the `~/.local/ax-v0.3.1/ax` build calls an `UpdateTask` RPC the deployed
|
||||
ax-server answers `Unimplemented` (client/server version mismatch); (c) the store endpoint
|
||||
`files.because-security.com` / `sast-memory` passes the egress check. **Remaining blocker (AX/cluster-internal,
|
||||
operator-owned):** dispatched tasks reach the `ousast-pool` workers and the replay actors execute and checkpoint,
|
||||
but the result never returns to the coordinator and no completion is recorded. `ousast-pool` logs show a gVisor
|
||||
cleanup failure: `runsc delete ... connecting to control server at PID NNN: connection refused` and `failed to
|
||||
terminate workload after checkpoint ... runsc delete: exit status 128`. Until the completion / checkpoint-cleanup
|
||||
path is fixed, the ax lane cannot produce results; the G0 baseline (7.2) was taken on the **docker lane**
|
||||
instead. The ax lane must be restored before it is the baseline/G1 lane (the stated target).
|
||||
|
||||
- **Engine coverage gap is comparison, not deadline (2026-10-11, preliminary n=1).** The cold-cache default arm
|
||||
(30s) times out the Joern CPG build → `quick_only`. But a long-deadline sample (`--arm long_deadline`, 300s) on a
|
||||
vulnerability change ran **206s with no timeout and still did not cover** (`coverage_status: incomplete`,
|
||||
`quick_only`): the reasons were `ambiguous_line_correspondence`, `unmatched_added_deleted_path_correspondence`,
|
||||
`eligibility_provenance_mismatch`, `witness_unresolved`, `comparison_unknown` — i.e. the engine could not
|
||||
correspond the replayed diff's lines between base and head. So **calibration / a longer deadline alone will not
|
||||
give engine coverage**; the blocker is the engine's base↔head comparison on real replayed diffs (consistent with
|
||||
the earlier "engine near-blind to fixes" slice-leak audit). This is G2 engine-fix territory (comparison /
|
||||
line-correspondence), not a G0 deadline knob. To confirm beyond n=1, the long-deadline arm must run at scale
|
||||
(needs the kube lane). Implication: the record-07 baseline is measured on the **quick-rule tier** (the default
|
||||
arm), and the engine arm's value is gated on fixing the comparison path, not on calibration.
|
||||
|
||||
- **Line-correspondence fix made, but it is necessary-not-sufficient (2026-10-11).** Root cause confirmed in
|
||||
`src/openultrasast/push/snapshot.py`: equal-region base↔head line pairs were mapped only when the line content
|
||||
was *globally unique in the file*, so common lines (`}`, blanks, imports) tripped `ambiguous_line_correspondence`
|
||||
and the finding's anchor was discarded → `comparison_unknown` → finding suppressed. Fixed to **run-aware
|
||||
positional anchoring**: a pair maps when its content occupies exactly one contiguous equal run per side (intra-run
|
||||
duplicates are forced/safe; cross-run duplicates stay withheld — soundness preserved, `test_push_snapshot.py`
|
||||
moved-operation guard still passes). 163 push tests pass. BUT live-testing the fix (mounted over the installed
|
||||
package, 300s) on a real messy change (a 14-file Java diff) did **not** flip coverage: it stayed `incomplete`
|
||||
because the binding constraints are elsewhere — `capability_unavailable` (empty capability registry always blocks
|
||||
`coverage_status=complete` until Task 8.3), the engine produced **no actionable defect at all**
|
||||
(`actionable_defect_ids: []`, a detection gap, not correspondence), plus `eligibility_provenance_mismatch` and
|
||||
`contributor-scan ... projection_unavailable`. So "near-blind to fixes" is **multi-causal**; the correspondence
|
||||
fix removes one real cause but engine coverage on unseen diffs also needs the capability registry (Task 8.3), the
|
||||
engine's actual vuln detection, and the eligibility/projection paths. Validating the correspondence fix's benefit
|
||||
needs a change where correspondence is the sole blocker (a small single-file diff), or the engine arm at scale.
|
||||
|
||||
## Open Questions
|
||||
|
||||
Maintainer decisions of 2026-10-04 resolve questions 1, 2, 4 and 5; question 3 stays open for G3.
|
||||
|
||||
1. **Resolved 2026-10-04.** Licences: only permissively licensed repositories are published by name. Copyleft and
|
||||
all other licences go to the gitignored private manifest, covered in the committed manifest by an opaque id and
|
||||
a digest (Req 1.4 as written; decision 1.4). Population v2's OSI-approved bar is not used.
|
||||
2. **Resolved 2026-10-04.** Concurrency stays at two AX lanes plus the VM lane. No operator placement change is
|
||||
requested (5.3).
|
||||
3. Per-family BLOCK (G3) needs about 100 changes per family. Pool slices together, or draw a family-enriched pool
|
||||
for G3?
|
||||
4. **Resolved 2026-10-04.** An adoption requires no significant regression on the unseen slice, not only a report
|
||||
(Req 5.4, amended; decision 7.3).
|
||||
5. **Resolved 2026-10-04.** G1 adds a registered long-deadline arm (`--deadline 300`) next to the default 30 s, so
|
||||
the table shows both what the user sees and what the engine could catch (decision 4.0, increment 8).
|
||||
@@ -1,107 +0,0 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
The safety net is meant for repositories it has never seen. This spec builds the instrument that measures that: a
|
||||
frozen pool of unseen repositories, a push-level metric (catch rate and false-alarm rate as the hook reports them),
|
||||
and an execution path that runs the replays as AX Tasks on the kube-ax cluster. Context: `brief.md`, the roadmap's
|
||||
G0-G1 (`.kiro/steering/roadmap.md`).
|
||||
|
||||
## Boundary Context
|
||||
|
||||
- **In scope**: pool selection and freezing; change extraction (vulnerability-introducing and ordinary commits);
|
||||
replay of each change through `ousast pre-push` and through registered experiment arms; the metric and its
|
||||
confidence intervals; AX Task execution with kind/VM fallback; the baseline for today's hook; rotation of pool
|
||||
slices.
|
||||
- **Out of scope**: any detection change; population v3 (stays reserved and unread); model calls inside the cluster.
|
||||
- **Adjacent expectations**: `learned-decision-engine` experiments use this pool as their primary unseen-repo score;
|
||||
`pre-push-safety-net` owns the hook being replayed; `plane-on-kubernetes` and the AX executor
|
||||
(`benchmarks/learn/engine_trace_ax.py`) supply the execution pattern; the reserved-repository guard covers every
|
||||
population and this pool.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: A pool no development step has touched
|
||||
|
||||
**User Story:** As the maintainer, I want the evaluation repositories to be ones nothing in development has seen, so a
|
||||
good number means the approach generalises.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. A repository is eligible only if it appears in no pair corpus, no population (v1, v2, v3), no harvest record, no
|
||||
memory example, no experiment unit and no measurement record. Eligibility is checked by code against every source
|
||||
before the pool is frozen; v3 is checked through the existing guard without being opened.
|
||||
2. The pool is frozen (a committed manifest with repository URLs, commit SHAs, licences and the freeze digest) before
|
||||
any change in it is replayed. A frozen pool is never edited; a correction is a new pool.
|
||||
3. Pool repositories are referenced only inside the pool's own directory (like `benchmarks/independent/`), and the
|
||||
reserved-repository guard is extended to them.
|
||||
4. Only permissively licensed repositories are published by name; others may be used with their names kept out of
|
||||
the repository (pointers in a local, gitignored manifest), recorded as such.
|
||||
|
||||
### Requirement 2: Two kinds of change per repository
|
||||
|
||||
**User Story:** As the maintainer, I want both vulnerable and ordinary pushes from the same repositories, so the
|
||||
false-alarm rate is measured on the code the catch rate is measured on.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. Vulnerability-introducing changes: from public advisories with a linked fix, the commit range that introduced the
|
||||
vulnerable code where it can be determined, else the fix's parent compared with an earlier base; each labelled
|
||||
with family and the vulnerable function. Post-model-cutoff advisories are preferred and flagged.
|
||||
2. Ordinary changes: commits from the same repositories' histories with no security fix, revert or advisory link,
|
||||
sampled across time and size; each recorded with its changed files and lines.
|
||||
3. Per repository, at least one vulnerability-introducing change and several ordinary changes; the pool's totals are
|
||||
sized so the false-alarm rate's 95% interval is no wider than +/- 3 points and the catch rate's no wider than
|
||||
+/- 10 points (sizes computed in the design).
|
||||
|
||||
### Requirement 3: The push-level metric
|
||||
|
||||
**User Story:** As the maintainer, I want the score to be what a user would see, so the number means something.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. Each change is replayed as `ousast pre-push --base <base> --head <head>` with the hook's default settings, in a
|
||||
clean checkout, with the record of every skipped check kept.
|
||||
2. Catch: a vulnerability-introducing change counts as caught when the hook (or the arm) reports a finding of the
|
||||
right family on the vulnerable function or its changed lines. False alarm: an ordinary change counts as a false
|
||||
alarm when the hook reports any advisory finding.
|
||||
3. Reported per family and overall: catch rate, false-alarm rate, and both with 95% Wilson intervals; plus coverage
|
||||
(changes where the hook could analyse the changed language) so a silent zero is visible.
|
||||
4. Arms (decision engine, combiner, engine evidence, fix mechanisms) are scored on the same replays: catch rate at
|
||||
fixed false-alarm budgets (1%, 5%, 10%), so arms are compared at equal noise.
|
||||
|
||||
### Requirement 4: Execution on kube-ax, designed for AX
|
||||
|
||||
**User Story:** As the maintainer, I want the replays to run on the cluster, so measurements are fast and the VM is
|
||||
free.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. One replayed change is one AX Task, named with the `ousast-engine-` prefix the operator's egress watcher matches,
|
||||
using an image built `FROM ghcr.io/norandom/ax-task-runner:v0.3.1` with this package, Joern and php-cli, pinned by
|
||||
digest.
|
||||
2. The task fetches the repository at the needed commits itself over HTTPS (public hosts are reachable from actors)
|
||||
and writes its record through a presigned link; no API keys or store credentials enter the cluster.
|
||||
3. A generic AX batch dispatcher serves both this replay and the existing engine passes: items to Tasks, at most the
|
||||
cluster's safe concurrency (two heavy tasks per worker), retry once on sandbox death, then a VM or kind fallback
|
||||
lane; progress and resume as in the engine runner.
|
||||
4. Arms that need model calls take the cluster's records and make their calls on the VM, within registered budgets.
|
||||
5. kind runs the same tasks unchanged as the local fallback.
|
||||
|
||||
### Requirement 5: Baseline, rotation and use
|
||||
|
||||
**User Story:** As the maintainer, I want a baseline now and a fresh slice for each decision, so the pool is not
|
||||
worn out by tuning.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. The first use is a baseline of today's hook on the first slice; its record is committed (counts and digests only,
|
||||
no repository names outside the pool directory).
|
||||
2. A slice that informed a change can no longer qualify that change; each adoption decision draws a new, untouched
|
||||
slice, recorded with its freeze digest.
|
||||
3. Every learned-decision-engine experiment from now on reports its unseen-repo score from this pool next to its
|
||||
held-out-fold score.
|
||||
4. An arm is adopted only if, on the untouched slice that qualifies it, it shows no significant regression against
|
||||
the current production arm: catch at the 5% false-alarm budget not significantly lower, and for an arm with one
|
||||
operating point the false-alarm rate not significantly higher (exact paired tests, two-sided, alpha 0.05).
|
||||
Reporting the unseen score alone is not enough to adopt. *(amended 2026-10-04, maintainer decision)*
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"feature_name": "unseen-repo-evaluation",
|
||||
"created_at": "2026-10-04T18:00:00Z",
|
||||
"updated_at": "2026-10-04T20:00:00Z",
|
||||
"language": "en",
|
||||
"phase": "implementation",
|
||||
"approvals": {
|
||||
"requirements": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"design": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"tasks": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
}
|
||||
},
|
||||
"ready_for_implementation": true
|
||||
}
|
||||
@@ -1,402 +0,0 @@
|
||||
# Implementation Plan
|
||||
|
||||
One task group per increment of `design.md` section 8, in that order. A group starts only after the previous
|
||||
group's exit is recorded. Maintainer decisions of 2026-10-04 are folded in: a registered long-deadline arm
|
||||
(`--deadline 300`) beside the hook's default 30 s (design 4.0, group 8); adoption needs no significant regression
|
||||
on the unseen slice (Req 5.4 as amended, design 7.3, tasks 6.4 and 8.2); only permissively licensed repositories
|
||||
are published by name, everything else goes to the gitignored private manifest under a digest (Req 1.4, design
|
||||
1.4); concurrency stays at two AX lanes plus the VM lane (design 5.3).
|
||||
|
||||
**2026-10-11 amendments folded in** (see design.md "Amendments (2026-10-11)"):
|
||||
|
||||
- **Pool p1 is frozen.** `benchmarks/unseen/pool-p1.toml` = 100 repos, slice 1; `freeze-p1.json` guard=0,
|
||||
dropped=0, deterministic (manifest `6bd2fa5d`); archived to `ousast-corpus/unseen/20260601/pool/`. Groups 1-3 are
|
||||
effectively done (the draw, draft and freeze ran end to end); formal records 02/03 were not written as separate
|
||||
`record.json` files — the freeze was run directly (`bisect_guard` then `freeze`). Freeze recipe: `bisect_guard
|
||||
--root $PWD --draft $CACHE/journal/draft-p1.toml` then `freeze --root $PWD --used-report <FILE>` where the file
|
||||
is the draw report's `used_set` object (not the whole report).
|
||||
- **Persistence is bucket-first** (`ousast-corpus`, `benchmarks/unseen/bank.py`); the local journal is a
|
||||
within-process cursor only. Raw per-change replay records go to **RustFS**, not `~/ousast-results/` on local disk
|
||||
— amend tasks 5.3/7.2's `--out` accordingly.
|
||||
- **Group 5 image note:** the replay image is the CI-built `ghcr.io/norandom/ousast-replay-task` (+ engine-task),
|
||||
publicly pullable by digest — not a local export-HEAD build and not `ousast-runner`. Task 7.1's "build and push
|
||||
the replay image" is satisfied by the CI `engine-image.yml` run; use that digest.
|
||||
- **Record 07 is a machine-label baseline** (labels SZZ-derived, 100 unchecked); state the caveat in the record.
|
||||
- **Baseline lane (7.2):** run on kube (`--lane ax`). The `ax` binary and the AX-lane store endpoint are
|
||||
operator-owned infra to stand up before 7.2; do not assume the cluster/network state.
|
||||
|
||||
Execution rules:
|
||||
|
||||
- **Who does what.** Codex implements every coding task offline: no network, no model, no store, no cluster. The
|
||||
coordinator runs every networked step (GitHub and OSV API calls, git fetches and clones, the S3 store, image
|
||||
pushes, kind and kube-ax). Those tasks are marked **(coordinator, network)**. Steps that need the coordinator but
|
||||
no network (agent label reads, adjudication, the v3 guard bisection, commits) are marked **(coordinator)**.
|
||||
Steps that spend model money are marked **paid: needs a budget go-ahead**. Each starts only after the maintainer
|
||||
confirms a ceiling against the provider balance read just before it.
|
||||
- **Done means.** A coding task is done when its tests pass in the host suite, `ruff check`, `ruff format --check`
|
||||
and `mypy` are clean, and the reserved-repository guard is green. No test makes a network, model or cluster call.
|
||||
- **Names stay in the pool directory.** Pool repository names and URLs appear only under `benchmarks/unseen/`
|
||||
(copyleft and other non-permissive ones only under the gitignored `benchmarks/unseen/private/`). Raw per-change
|
||||
records live in `~/ousast-results/unseen/` and the store, never in the repository.
|
||||
- **Population v3 is never opened.** It is checked only through the guard test, run as a black box with its
|
||||
output discarded.
|
||||
- **Every group ends with a measured exit** and a record at
|
||||
`benchmarks/measurements/<date>-unseen-NN-<slug>/record.json`: commit, image digests where used, the instrument
|
||||
checks (bytes read, JVM time, exit codes), counts and digests. No repository names. The guard sweeps these
|
||||
records.
|
||||
- **Instrument first** (AGENTS.md). A zero, an empty draft or a fast run is treated as an instrument failure until
|
||||
bytes read and wall time prove otherwise.
|
||||
|
||||
- [x] 1. Used set and guard (design 1.2, 6; increment 1)
|
||||
- [x] 1.1 Used-set builder
|
||||
- `benchmarks/unseen/eligibility.py`: normalises names (lower case, no `.git`, no trailing slash) and builds the
|
||||
used set from every source of design table 1.2: pair catalogs, recipes and datasets under `benchmarks/pairs/`
|
||||
plus the local pointer-pair manifests as `benchmarks/pairs/catalog_gen.py` reads them; `population-v1.toml`
|
||||
and `population-v2.toml`; the advisory-fix batch catalogs; memory `example` rows (`repo` field) through an
|
||||
injected `MemoryStore` (`FileStore` locally, the S3 store when the coordinator passes one); experiment units
|
||||
under `plane/experiments/`; `benchmarks/measurements/` and `benchmarks/experiments/`; a text sweep for
|
||||
`github.com/<owner>/<name>` over `benchmarks/`, `src/`, `tests/`, `plane/` and `.kiro/`. It never reads
|
||||
population v3 and never prints a name. Output: counts per source and the sha256 of the sorted set.
|
||||
- `tests/test_unseen_eligibility.py`: a synthetic tree with one repository per source, each found; case, `.git`
|
||||
and trailing-slash variants collapse; stdout and the output JSON contain no name; a v3-shaped file planted in
|
||||
the synthetic tree is not opened (open is patched to fail on it).
|
||||
- _Requirements: 1.1_
|
||||
- [x] 1.2 Canonical resolution, renames and forks
|
||||
- `eligibility.py`: `resolve(candidate, api)` with an injected GitHub client returns the canonical `full_name`,
|
||||
former names (redirects) and the fork network (`parent`, `source`). A candidate is ineligible when any of them
|
||||
is in the used set; one candidate per fork network is kept.
|
||||
- Tests with a fake client: a planted used repository, a renamed one and a fork of a used one are each
|
||||
rejected; two forks of one network keep one; the rejection reason is a source label, not a name.
|
||||
- _Requirements: 1.1_
|
||||
- [x] 1.3 Guard extended to pools
|
||||
- `tests/test_independent_population.py`: the sweep becomes a list of (reservation directory, names):
|
||||
`benchmarks/independent/` for populations, `benchmarks/unseen/` for pools (public manifests, plus
|
||||
`benchmarks/unseen/private/*.toml` when present). The sweep also covers `plane/`. New tests: pool and
|
||||
population names are disjoint (failure prints a count only); a frozen pool's manifest digest equals its
|
||||
`freeze-pN.json` (skipped while no pool is frozen). The existing v3 handling is unchanged.
|
||||
- Each reservation is exempt only inside its own directory: population names (v3 included) are still swept in
|
||||
`benchmarks/unseen/`, the gitignored `private/` files included. Task 3.1 depends on this.
|
||||
- `.gitignore`: `benchmarks/unseen/private/`.
|
||||
- Evidence: the guard is green with an empty `benchmarks/unseen/`, and red when a test fixture names a pool
|
||||
repository in `benchmarks/measurements/` (shown once in the record).
|
||||
- _Requirements: 1.2, 1.3, 1.4_
|
||||
- [x] 1.4 Usage ledger
|
||||
- `benchmarks/unseen/ledger.py`: append-only `usage-pN.jsonl`, one row per use (slice, decision or experiment
|
||||
id, purpose `baseline | exploratory | informed | qualifies`, date, result digest, freeze digest). It refuses a
|
||||
`qualifies` row for a change whose slice already has an `informed` row for it, and refuses to rewrite rows.
|
||||
- `tests/test_unseen_ledger.py`: append, the refusal, a rewritten row detected by a digest chain.
|
||||
- _Requirements: 5.2_
|
||||
- [x] 1.5 Used-set measurement and record 01 **(coordinator, network)**
|
||||
- Run `eligibility.py` against the real sources, the S3 memory store included (read only).
|
||||
- Instrument: each source reports files read and bytes; a source with zero rows fails the run unless it is
|
||||
known empty.
|
||||
- `benchmarks/measurements/<date>-unseen-01-used-set/record.json`: counts per source, used-set digest, guard
|
||||
result (green on the empty pool), test counts of 1.1-1.4.
|
||||
- _Requirements: 1.1, 1.3_
|
||||
|
||||
- [ ] 2. Sourcing and extraction (design 1.1, 1.3, 1.4, 2, 3; increment 2)
|
||||
- [x] 2.1 Advisory candidates and repository filters
|
||||
- `benchmarks/unseen/source.py`: from GHSA advisory JSON (pip, npm, maven, composer; published 2019 or later)
|
||||
keep those whose CWE maps to a family through `src/openultrasast/model/taxonomy.py` and that link one fix
|
||||
commit with one parent in one repository; attach the OSV first affected version; flag `post_cutoff` when
|
||||
published after 2026-06-01. Repository filters: anonymous HTTPS, at most 500 MB, licence at the pinned head
|
||||
(LICENSE file, then the API's SPDX id) classed `permissive` (the design 1.4 list) or `private`.
|
||||
- `tests/test_unseen_source.py` on recorded fixture JSON (no network): CWE mapping, multi-parent and
|
||||
multi-repository fixes dropped, cutoff flag, licence classing (GPL, AGPL, LGPL, none and source-available go
|
||||
private).
|
||||
- _Requirements: 1.4, 2.1_
|
||||
- [x] 2.2 Vulnerability-introducing range
|
||||
- `benchmarks/unseen/extract.py`: the `introducing` method of design 2.1 (blame of the sink line, or the
|
||||
declaration for an absence bug; absence at V's first parent; no merge or root; at most 50 files and 2,000
|
||||
lines; OSV version bracket when present; walk-back of at most five steps) and the `last_touch` fallback. Each
|
||||
change records method, family, vulnerable function resolved in head, head-side lines, `post_cutoff`, advisory
|
||||
id. Git only; no scanner or engine.
|
||||
- `tests/test_unseen_extract.py` on a synthetic repository built in a temp dir: accepted V; a reformat walked
|
||||
back; a merge rejected; size cap; bracket rejection; fallback chosen when no V passes.
|
||||
- _Requirements: 2.1_
|
||||
- [x] 2.3 Ordinary-commit draw
|
||||
- `extract.py`: first-parent, non-merge commits touching an analysed language; the exclusions of design 2.2
|
||||
(security words of the narrow pattern, reverts, bot and bump commits, manifest- or lock-only, generated or
|
||||
vendored, advisory fix commits, commits touching a known vulnerable function, more than 1,000 source lines,
|
||||
counted). Seeded draw of 12 per repository over three time thirds by four size buckets, proportional, at least
|
||||
one per third; a repository with fewer than 40 eligible commits is dropped.
|
||||
- Tests: each exclusion; the narrow pattern keeps "query", "token", "session"; allocation and the seed
|
||||
reproduce the same draw; the 40-commit floor.
|
||||
- _Requirements: 2.2, 2.3_
|
||||
- [x] 2.4 Draft manifest and slices
|
||||
- `benchmarks/unseen/draft.py`: writes `benchmarks/unseen/draft-p1.toml` (permissive entries by name) and
|
||||
`benchmarks/unseen/private/draft-p1.toml` (the rest); the public file holds an opaque id and
|
||||
sha256(URL, commits) per private entry. Seeded shuffle into three slices of 100 repositories, stratified by
|
||||
ecosystem and `post_cutoff`; each repository has one introducing change and 12 ordinary changes.
|
||||
- Tests: the digest covers private entries; no private URL in the public file; the same seed gives the same
|
||||
slices; per-slice strata within one repository of proportional.
|
||||
- _Requirements: 1.2, 1.4, 2.3, 5.2_
|
||||
- [ ] 2.5 Sourcing run **(coordinator, network)**
|
||||
- `gh api /advisories` enumeration, OSV lookups, canonical resolution against the used set (1.2), blobless
|
||||
clones into a scratch cache, extraction (2.2, 2.3), draft (2.4).
|
||||
- Instrument: clones counted with bytes on disk; a repository whose clone reads zero source files is an
|
||||
instrument failure, not a drop. Fewer than 300 eligible repositories stops the run with counts; the
|
||||
eligibility bars are not relaxed to fill it.
|
||||
- Observable: a draft of 300 repositories (3,900 changes) on disk.
|
||||
- _Requirements: 1.1, 1.4, 2.1, 2.2, 2.3_
|
||||
- [ ] 2.6 Label check and record 02 **(coordinator)**
|
||||
- Every third `introducing` change read by an agent, no model call, marked `confirmed | refactor | unclear`
|
||||
in the draft.
|
||||
- `benchmarks/measurements/<date>-unseen-02-draft/record.json`: candidates per ecosystem, rejections per used
|
||||
source, licence split (public, private), method counts, post-cutoff counts, confirmed share, ordinary-commit
|
||||
exclusion counts. No names.
|
||||
- _Requirements: 2.1, 2.2_
|
||||
|
||||
- [ ] 3. Freeze p1 (design 1.2, 6; increment 3)
|
||||
- [x] 3.1 Guard bisection tool
|
||||
- `benchmarks/unseen/bisect_guard.py`: rewrites the draft with a candidate subset, runs
|
||||
`.venv/bin/pytest -q -p no:cacheprovider tests/test_independent_population.py -k referenced` with stdout and
|
||||
stderr discarded, reads only the exit code, and bisects until green. Reports the dropped count only.
|
||||
- `tests/test_unseen_bisect.py` with an injected fake guard (a hidden set): finds and drops exactly the hidden
|
||||
members; never logs a candidate; an exit code other than 0 or 1 aborts (instrument failure).
|
||||
- _Requirements: 1.1_
|
||||
- [x] 3.2 Freeze writer
|
||||
- `benchmarks/unseen/freeze.py`: canonical `pool-p1.toml` (and the private counterpart) from the bisected draft;
|
||||
`freeze-p1.json` with used-set digest and per-source counts, guard result, dropped count, label-check counts
|
||||
and the freeze digest (sha256 of the canonical manifest). Refuses to write over a frozen pool; a correction
|
||||
is `p2`.
|
||||
- Tests: digest stable under key order; overwrite refused; the guard's digest test (1.3) passes on a fixture
|
||||
pool.
|
||||
- _Requirements: 1.2_
|
||||
- [x] 3.3 Protocol p1
|
||||
- `benchmarks/unseen/protocol-p1.md`: the scoring rules of design section 4 (default flags, decision 4.0's
|
||||
long-deadline arm, catch and location rules, false alarm, coverage states), the statistics of section 3 and
|
||||
7, and the adoption gate of 7.3, stated before any replay.
|
||||
- Observable: the file exists and its sha256 is listed in `freeze-p1.json`.
|
||||
- _Requirements: 3.1, 3.2, 3.3, 3.4, 5.4_
|
||||
- [ ] 3.4 Freeze run and record 03 **(coordinator)**
|
||||
- Run the bisection (3.1), then the freeze (3.2). Guard green, digest test active.
|
||||
- `benchmarks/measurements/<date>-unseen-03-freeze/record.json`: freeze digest, protocol digest, dropped
|
||||
count, final counts per slice and kind.
|
||||
- _Requirements: 1.1, 1.2, 1.3_
|
||||
|
||||
- [ ] 4. Dispatcher (design 5.3; increment 4)
|
||||
- [x] 4.1 Items, workloads and the AX lane
|
||||
- `benchmarks/ax/batch.py`: `Item` (id, input objects, result object name, command, extra env, input digest),
|
||||
`Workload` (image digest, allowed URL env names, deadline, result validator) and the `ax` lane, moved from
|
||||
`benchmarks/learn/engine_trace_ax.py`: manifest render, egress readiness, presigned GET and PUT as `.dat`,
|
||||
apply, poll result object and AX phase, delete, object cleanup. The `kind` lane is the same lane with the
|
||||
kind context.
|
||||
- `tests/test_ax_batch.py` with a fake `ax` CLI and a fake store: manifest holds only the allowed URL env
|
||||
names and no credential; names carry the `ousast-engine-` prefix; cleanup on success and failure.
|
||||
- _Requirements: 4.1, 4.2, 4.5_
|
||||
- [x] 4.2 Scheduler, fallback and resume
|
||||
- `batch.py`: two AX lanes and one VM `docker` lane by default (one container at a time); sandbox death retried
|
||||
once under a new name, a second death or a JVM OOM goes to the VM fallback queue ahead of overflow;
|
||||
`progress.json` written by the scheduler only; a `STOP` file halts launches; resume skips items whose stored
|
||||
record has the same input digest; a cleanup failure stops dispatching.
|
||||
- Tests with fake lanes: retry then fallback; OOM straight to fallback; resume skips; `STOP`; cleanup failure
|
||||
halts; never more than two AX tasks in flight.
|
||||
- _Requirements: 4.3_
|
||||
- [x] 4.3 Engine pass on the dispatcher
|
||||
- `benchmarks/learn/engine_trace_ax.py` and `engine_trace.run_plan` become an engine `Workload` over `batch.py`;
|
||||
no behaviour change.
|
||||
- Observable: `tests/test_ax_executor.py` and `tests/test_engine_trace.py` pass with their assertions unchanged
|
||||
(moved imports only).
|
||||
- _Requirements: 4.3_
|
||||
- [ ] 4.4 Engine smoke on kube-ax and record 04 **(coordinator, network)**
|
||||
- Two pins through the new dispatcher on kube-ax, compared with their stored records.
|
||||
- `benchmarks/measurements/<date>-unseen-04-dispatcher/record.json`: image digest, per-pin wall time and JVM
|
||||
time, record digests equal to the stored ones (or the diff counted), cleanup result.
|
||||
- _Requirements: 4.3_
|
||||
|
||||
- [ ] 5. Replay task (design 5.1, 5.2; increment 5)
|
||||
- [x] 5.1 Image stage
|
||||
- `plane/Dockerfile.engine-task`: second stage `FROM engine AS replay` adding `git`, the package with its
|
||||
runtime dependencies and `benchmarks/unseen/replay_entry.py`; smoke `ousast pre-push --help`. The root image
|
||||
stays `ghcr.io/norandom/ax-task-runner:v0.3.1`; the engine stage is unchanged.
|
||||
- `tests/test_unseen_image.py`: parses the Dockerfile; the `replay` stage builds from `engine`; the root image
|
||||
pin; the engine stage's text is byte-identical to before.
|
||||
- _Requirements: 4.1_
|
||||
- [x] 5.2 Task entry
|
||||
- `benchmarks/unseen/replay_entry.py`, standard library only, reusing `engine_task_entry.transfer`: GET the spec;
|
||||
`git init` and depth-1 fetches of head and base over HTTPS (180 s bound), verified with `git cat-file -e`; run
|
||||
`ousast pre-push` with the spec's hook flags (defaults, or `--deadline 300` for the long-deadline arm), heap
|
||||
1700 MB and the cluster JVM flags; pack `push.json`, `push.txt`, `engine.json`, `instrument.json` (files and
|
||||
bytes in the checkout, bytes of each changed file, JVM wall time, hook exit code, step timings, image
|
||||
digest); PUT as `.dat`. Any error PUTs a failure record naming the step and status, never URL text.
|
||||
- `tests/test_unseen_replay_entry.py`: a local bare repository and a localhost presign stub; success record
|
||||
fields; a missing commit, a failed fetch and a hook crash each give a failure record; zero bytes read and a
|
||||
replay faster than JVM start are flagged `suspect_fast`.
|
||||
- _Requirements: 3.1, 4.1, 4.2_
|
||||
- [x] 5.3 Replay workload
|
||||
- `benchmarks/unseen/replay.py`: builds `Item`s from a frozen slice and an arm (`default` or `long_deadline`);
|
||||
the spec object carries the repository URL so names stay out of AX status and telemetry; task names
|
||||
`ousast-engine-replay-<slug>-<digest>` with an opaque slug; the validator rejects a record without
|
||||
`instrument.json`; records go to `~/ousast-results/unseen/<pool>/<slice>/<arm>/` and the store.
|
||||
- Tests: no repository name in any task name or manifest; only the two presigned URLs in the env; input
|
||||
digest changes with the arm's flags; `instrument_failure` items queued for one re-run.
|
||||
- _Requirements: 3.1, 4.1, 4.2, 4.3_
|
||||
- [ ] 5.4 Proof on kind, then kube-ax, and record 05 **(coordinator, network)**
|
||||
- Build and push the `replay` target, pinned by digest. Five changes from this repository's own history (never
|
||||
pool changes) on kind, then the same five on kube-ax, then on the VM lane.
|
||||
- Exit: instrument fields show bytes read and realistic JVM time on every lane; quick-tier findings equal
|
||||
across lanes.
|
||||
- `benchmarks/measurements/<date>-unseen-05-replay-task/record.json`: image digest, per-lane timings, bytes,
|
||||
exit codes, the quick-tier comparison.
|
||||
- _Requirements: 4.1, 4.2, 4.5_
|
||||
|
||||
- [x] 6. Scorer (design 3, 4, 7.1, 7.3; increment 6)
|
||||
- [x] 6.1 Coverage states
|
||||
- `benchmarks/unseen/score.py`: each record gets one state (`instrument_failure`, `unanalysable`, `quick_only`,
|
||||
`engine_covered`) from `instrument.json` and the artifact; `suspect_fast` goes to failure; a slice above 5%
|
||||
`instrument_failure` after re-runs is refused.
|
||||
- `tests/test_unseen_score.py` on synthetic records: every state; the refusal; a failure never counts as quiet
|
||||
or missed.
|
||||
- _Requirements: 3.3_
|
||||
- [x] 6.2 Catch and false-alarm rules
|
||||
- `score.py`: family match (engine family, quick-rule CWE through `taxonomy.py`), the location table of design
|
||||
4.1, all three streams with the stream named per catch; false alarm on any finding of any stream; the
|
||||
alerts-only rate.
|
||||
- Tests: right family wrong place misses; `access_control` on the handler; `config_secrets` file scope; a
|
||||
quick-rule-only catch; one advisory finding on an ordinary change is a false alarm.
|
||||
- _Requirements: 3.2_
|
||||
- [x] 6.3 Rates, intervals and breakdowns
|
||||
- `score.py`: Wilson 95% intervals; repository-cluster bootstrap for false alarms; ICC; breakdowns per family,
|
||||
method, `post_cutoff`, stream, coverage state and the confirmed subset.
|
||||
- Tests: the n values of design section 3 (315, 483, 756, 62, 88, 97) reproduced; bootstrap seeded; ICC on a
|
||||
synthetic clustered set.
|
||||
- _Requirements: 2.3, 3.3_
|
||||
- [x] 6.4 Budgets, paired tests, the gate and record 06
|
||||
- `score.py`: per-budget thresholds (1%, 5%, 10%), labelled `exploratory` in-slice and `fixed` when supplied;
|
||||
a binary arm's single operating point with the budgets it meets; exact two-sided McNemar for catch and for
|
||||
false alarm; `gate(A, B)` returning `pass | regression | not_run` per design 7.3.
|
||||
- Tests: thresholds on a synthetic score set; McNemar exact values; a binary arm with a significant
|
||||
false-alarm increase gives `regression`; missing slice or excess failures give `not_run`.
|
||||
- `benchmarks/measurements/<date>-unseen-06-scorer/record.json`: test counts and the digest of the scorer's
|
||||
output on the committed synthetic fixture.
|
||||
- _Requirements: 3.4, 5.4_
|
||||
|
||||
- [ ] 7. Baseline (design 4, 6; increment 7; G0 exit)
|
||||
- [ ] 7.1 Frozen source and dry run **(coordinator, network)**
|
||||
- Export HEAD to a frozen tree; build and push the `replay` image from it, pinned by digest.
|
||||
- `benchmarks/push/python_overhead.py` run with the stub engine to size per-replay Python cost; dispatcher dry
|
||||
run on slice 1 lists 1,300 items without launching.
|
||||
- _Requirements: 5.1, 4.1_
|
||||
- [ ] 7.2 Baseline replay of slice 1 **(coordinator, network)**
|
||||
- Default arm, 1,300 replays over two AX lanes and the VM lane (about 9 hours); `instrument_failure` items
|
||||
re-run once; the run stops for a cause above 5% failures.
|
||||
- Observable: 1,300 records in `~/ousast-results/unseen/p1/1/default/` and the store, each with
|
||||
`instrument.json`.
|
||||
- _Requirements: 3.1, 4.3, 5.1_
|
||||
- [ ] 7.3 Adjudication **(coordinator)**
|
||||
- Flagged ordinary changes adjudicated against protocol-v2's standard (all if 30 or fewer, else every third);
|
||||
the adjudicated share kept beside the false-alarm rate. ICC measured; above 0.05 noted as "next pool sizes
|
||||
up".
|
||||
- _Requirements: 3.2, 2.3_
|
||||
- [ ] 7.4 Record 07 and the ledger **(coordinator)**
|
||||
- `benchmarks/measurements/<date>-unseen-07-baseline/record.json`: `score.py` output (counts, rates, Wilson and
|
||||
bootstrap intervals, coverage per state, per family, method, stream), freeze digest, image digest, commit.
|
||||
`usage-p1.jsonl` row `baseline`.
|
||||
- Observable: guard green over the record; G0's exit met.
|
||||
- _Requirements: 5.1, 5.2, 3.3_
|
||||
|
||||
- [ ] 8. Arms on slice 1 (design 4.0, 5.4, 7.1-7.3; increment 8; G1)
|
||||
- [ ] 8.1 Registered long-deadline arm
|
||||
- `plane/experiments/` manifest for the long-deadline arm: hook flags with `--deadline 300`, budget 0,
|
||||
`unseen: {pool: p1, slice: 1}`, committed before its first replay. `src/openultrasast/learn/experiments.py`
|
||||
accepts `unseen` and a hook-flag arm.
|
||||
- `tests/test_learn_experiments.py`: the manifest loads; an `unseen` block without pool or slice is refused.
|
||||
- _Requirements: 3.4, 5.3_
|
||||
- [ ] 8.2 `unseen` block and adoption gate in `analyse`
|
||||
- `experiments.py` `analyse`: writes `unseen` into `result.json` (catch at each budget per arm, paired B - A,
|
||||
McNemar, `gate`) from `score.py`; checks the usage ledger; adoption requires the fold rule and
|
||||
`gate == pass`.
|
||||
- Tests: `regression` and `not_run` each block adoption; an `informed` slice cannot qualify; result JSON shape.
|
||||
- _Requirements: 5.2, 5.3, 5.4_
|
||||
- [ ] 8.3 Engine-evidence arm
|
||||
- `benchmarks/unseen/arms.py`: a score per change from `engine.json` (witness rows on head-side changed
|
||||
functions), no model; plugs into the budget thresholds of 6.4.
|
||||
- Tests on synthetic records: score monotone in witness strength; a change without engine coverage gets no
|
||||
score and is reported, not counted quiet.
|
||||
- _Requirements: 3.4_
|
||||
- [ ] 8.4 Model-arm adapter
|
||||
- `arms.py`: candidates from head-side changed functions of stored records, scored by the current decision
|
||||
engine program (and the combiner when its program exists; else its row reads `not_built`); change score is the
|
||||
maximum; metered client, dry run first, response cache, the manifest's `budget_usd` as the hard ceiling.
|
||||
- Tests with a scripted client: dry-run estimate, ceiling stop, cache replay; no model call.
|
||||
- _Requirements: 4.4, 3.4_
|
||||
- [ ] 8.5 Long-deadline replay of slice 1 **(coordinator, network)**
|
||||
- 1,300 replays of the `long_deadline` arm, bounded at about 345 s each (up to about 42 hours over three
|
||||
lanes), with `STOP` and resume across sessions; failures re-run once.
|
||||
- Observable: 1,300 records under `~/ousast-results/unseen/p1/1/long_deadline/`, JVM time in each.
|
||||
- _Requirements: 3.1, 3.4, 4.3_
|
||||
- [ ] 8.6 Model arms on slice 1 **(coordinator, network; paid: needs a budget go-ahead)**
|
||||
- Dry run gives the candidate count; estimate at the injection slice's measured cost (about $0.008 per
|
||||
candidate at k = 5); the maintainer sets the ceiling against the balance read just before. Then the decision
|
||||
engine arm (and the combiner if built) on the VM.
|
||||
- _Requirements: 4.4, 3.4_
|
||||
- [ ] 8.7 Record 08: the arms table **(coordinator)**
|
||||
- `benchmarks/measurements/<date>-unseen-08-arms/record.json`: per arm (hook 30 s, hook 300 s, engine evidence,
|
||||
decision engine, combiner or `not_built`) catch at 1/5/10% budgets with Wilson intervals and realised
|
||||
false-alarm rates, coverage per state, spend; `exploratory` rows in `usage-p1.jsonl`.
|
||||
- Observable: one committed table of arms on slice 1; G1's table exists.
|
||||
- _Requirements: 3.4, 5.2, 5.3_
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- 2026-10-04: Tasks 1.1–1.4 implemented offline and independently reviewed. Task 1.5 remains for the
|
||||
coordinator; no networked measurement or commit was performed. Command and API contracts:
|
||||
`benchmarks/unseen/README.md`. Offline evidence:
|
||||
`benchmarks/measurements/2026-10-04-unseen-01-offline/record.json`.
|
||||
- Population guard scope is preserved under the maintainer's hard rule: benchmarks/src/tests for
|
||||
populations, plus plane for pools. Each directory exemption is reservation-specific, including private
|
||||
pool manifests. Extending population scope to plane exposed 30 existing references; no historical
|
||||
records were changed or exempted inside the existing population scope.
|
||||
- S3 inventory opens through `open_store(read_only=True)` because ordinary store construction writes a
|
||||
Select capability probe. Default store behavior is unchanged; inventory uses LIST/GET only.
|
||||
- The freeze writer must match the canonical TOML digest encoding documented in the pool README and
|
||||
exercised by `test_independent_population.py`; final pool discovery does not trust mutable status.
|
||||
|
||||
### Group 2 implementation evidence (2026-10-04)
|
||||
|
||||
Tasks 2.1–2.4 are implemented offline and independently reviewed. Task 2.5 is
|
||||
`python -m benchmarks.unseen.draft --root "$PWD" --s3-store s3:// --seed 20260601`
|
||||
with the documented PATH/PYTHONPATH and GH_TOKEN environment. No networked draw,
|
||||
label check, group-2 measurement record, guard bisection, or commit was performed.
|
||||
The group header remains pending its coordinator exits 2.5/2.6.
|
||||
|
||||
The command rebuilds the group-1 used set, resolves renamed/fork repositories,
|
||||
uses disposable external blobless clones, re-reads licences at extracted heads,
|
||||
and emits counts/digests only. The README documents injected clients, method bias,
|
||||
the hard 300-repository floor, and an explicitly unmeasured resource estimate.
|
||||
Independent review caught and fixed ambiguous parent-function absence and stale
|
||||
candidate-404 handling; regression tests cover both.
|
||||
|
||||
Validation: requested pytest selection 103 passed, 1 skipped, exit 0 (population-test
|
||||
output discarded); `ruff check .` exit 0; `ruff format --check .` exit 0 (411 files);
|
||||
`mypy src/openultrasast` exit 0 (160 source files). The CLI `--help` smoke exited 0.
|
||||
Source-read instrumentation printed 20,101 bytes for the extraction implementation.
|
||||
|
||||
### Groups 4–6 offline implementation (2026-10-04)
|
||||
|
||||
Tasks 4.1–4.3, 5.1–5.3 and 6.1–6.4 are implemented and independently reviewed.
|
||||
Tasks 4.4 and 5.4 remain coordinator-owned network proofs; group headers 4 and 5 stay
|
||||
pending those exits. No commit, network request, Docker, AX or kubectl command was run.
|
||||
The sourcing job's draft.py, source.py, extract.py and journal.py are unchanged.
|
||||
|
||||
The shared dispatcher retains the engine adapter's historical questions.json object
|
||||
name to preserve its unchanged assertions; replay uses spec.dat and result.dat.
|
||||
The engine Dockerfile body is unchanged except its required AS engine alias; CI builds
|
||||
engine and replay separately. S3Store's read_only class default preserves existing
|
||||
constructor-bypassing transport fixtures; initialized read-only stores still refuse writes.
|
||||
|
||||
Exact coordinator commands and input contracts are in benchmarks/unseen/README.md,
|
||||
including frozen TOML/private pointer verification, scoring joins, own-history proof
|
||||
preparation and anonymous measurement-record generation. JVM wall time is the hook's
|
||||
aggregate scan build/query timing and is labelled as such. Unsupported-only partitions
|
||||
with explicit language_not_covered evidence do not pretend to have launched a JVM.
|
||||
|
||||
Record 06 contains the committed synthetic fixture and scorer-output digests. The
|
||||
reserved-repository guard was run as a black box with both output streams discarded.
|
||||
Independent review: APPROVED; all reported integration and coverage defects resolved.
|
||||
|
||||
Final verification: requested pytest command 279 passed, 8 skipped, exit 0; separate
|
||||
`tests/test_ax_batch.py` 5 passed, exit 0; `ruff check .`, `ruff format --check .`,
|
||||
`mypy src/openultrasast`, and the output-discarded reserved-repository guard all exit 0.
|
||||
The own-history proof preparer and replay dry run read five items and exit 0.
|
||||
@@ -1,32 +0,0 @@
|
||||
# kube-ax operator constraints
|
||||
|
||||
Verified by the operator on 2026-10-05. These constraints apply across all workloads.
|
||||
|
||||
- Two workers, one task per worker, at most two tasks in flight cluster-wide. AX has no
|
||||
queue: a third apply returns `no free workers available`. Treat this as back-pressure,
|
||||
retry the same task/lane with 15–30 seconds of jitter until its deadline, and never
|
||||
count it as a sandbox failure or trigger VM fallback. The host dispatcher shares a
|
||||
two-slot reservation across workload instances and long-lived search executors.
|
||||
- `/` and `/tmp` are RAM-backed and consume the worker's 3 GiB. `/workspace` is the only
|
||||
disk-backed path, with about 2 GiB usable per task. Executor and verifier entrypoints
|
||||
use `/workspace/tmp`, `/workspace/.npm`, `/workspace/.m2`, `/workspace/.composer`,
|
||||
`/workspace/.pip`, and `/workspace/.gradle` through the respective tool environment
|
||||
variables. Builds and extracted checkouts live under `/workspace` too. The default
|
||||
aggregate workspace scratch guard is 2 GiB; `OUSAST_SCRATCH_BYTES` configures it.
|
||||
Exceeding it returns `could_not_build`, reason `scratch limit`. The guard polls
|
||||
during child execution and checks after exit; it is not an instantaneous filesystem
|
||||
quota or a replacement for the worker memory limit.
|
||||
- `ousast-engine-search-verify-*` tasks have no egress. The requested exception for
|
||||
`files.because-security.com:443` is **pending**, not operationally verified. They
|
||||
receive a presigned manifest for a gzip level-6 archive containing `checkout/`,
|
||||
`products/`, and `spec.json`, built/exported by an executor. Compressed archives
|
||||
retain the 1 GiB cap and travel in parts of at most 90 MB, each digest-checked
|
||||
before assembly and extraction under `/workspace`. Only `VERIFY_INPUT_URL` and `RESULT_URL` cross the
|
||||
verifier boundary. Verifiers reject build recipes and do not install dependencies
|
||||
or fetch repositories. Executors finish and release their slots before verifier
|
||||
repetitions start. Both task types must be provisioned without model credentials.
|
||||
- All tasks share one public IP. Repository acquisition uses HTTPS git clone from
|
||||
`github.com` or archives from `codeload.github.com`, never `api.github.com`. Host-side
|
||||
metadata queries are a separate concern. Search tasks receive checkout archives;
|
||||
executor tool guidance carries the same acquisition rule. Retain the 60-second
|
||||
first-download retry for delayed task egress policy activation.
|
||||
@@ -1,60 +0,0 @@
|
||||
# Operator notifications (Discord)
|
||||
|
||||
Recorded 2026-10-09 from the maintainer's direction. OpenUltraSAST notifies the operator on
|
||||
Discord about the status and results of its own processes, and especially when a process is
|
||||
**stuck or otherwise impaired**. This is an operator channel, not a developer or infrastructure
|
||||
one.
|
||||
|
||||
## Audience
|
||||
|
||||
The **system operator** and the **AI operator** who run and oversee the tool. Not developers
|
||||
debugging the code, and not an infrastructure on-call. The messages answer "is the work moving,
|
||||
did it finish, what did it produce, is it stuck" for someone supervising a run, not "why does this
|
||||
function throw".
|
||||
|
||||
## Scope: OpenUltraSAST processes and their results
|
||||
|
||||
Notify on the actual OpenUltraSAST workloads and what they produce:
|
||||
|
||||
- scans and pre-push checks (a completed check and its verdict/degradations),
|
||||
- unseen-repo sourcing / draft, freeze, and replay / baseline runs,
|
||||
- learning and evaluation runs (compile, evaluate, experiments) and their recorded results,
|
||||
- fusion / adjudication and any long-running job.
|
||||
|
||||
**Not** k3s or ax cluster infrastructure. Cluster, node, and ax-task health is a separate
|
||||
infrastructure concern monitored elsewhere; this channel does not carry it. The line is: a message
|
||||
here is about an OpenUltraSAST process or result, never about the plane's plumbing.
|
||||
|
||||
## When to notify
|
||||
|
||||
- **Completion:** a process finished, with a one-line result summary (counts, status, digest).
|
||||
- **Stuck or impaired (the important case):** a stall, a clean-but-short result
|
||||
(`insufficient_*`), an abort or `instrument_failure`, a timeout or deadline hit, budget
|
||||
exhaustion, or a resource limit (disk, memory). These @mention the operator so a supervised run
|
||||
does not sit silently broken.
|
||||
|
||||
Advisory and low-volume by design: a start/finish and the impaired cases, not per-step chatter.
|
||||
|
||||
## Transport and secrets
|
||||
|
||||
A Discord webhook, with credentials referenced by name only and stored in gitignored `.env`, like
|
||||
every other credential (`.env.example` lists the names, blank):
|
||||
|
||||
- `DISCORD_WEBHOOK_URL` — the operator channel webhook.
|
||||
- `DISCORD_OPERATOR_ID` — the account to @mention on a stuck/impaired alert.
|
||||
|
||||
The webhook URL is a secret (anyone holding it can post to the channel): it never enters a tracked
|
||||
file, a prompt, a log, or a commit, and it is rotated if exposed. Notifications are **off** when the
|
||||
webhook is unset, so the tool runs the same with or without it.
|
||||
|
||||
## Content discipline
|
||||
|
||||
Messages carry status, counts, and digests only, under the same redaction as the hardening and
|
||||
measurement-record conventions: no exploit payloads or request strings, no secrets, no customer
|
||||
content, no repository contents. A notification is safe to read in a shared channel.
|
||||
|
||||
## Status
|
||||
|
||||
Policy recorded; the notifier (reading the env-named webhook, quiet when unset, honoring the
|
||||
redaction above) is to be wired into the process entry points and may be specced separately. Until
|
||||
then this steering states the intended behavior for any process that adds notifications.
|
||||
@@ -1,23 +1,8 @@
|
||||
# Overview — what this is, and what is actually true of it
|
||||
|
||||
## Current state, 2026-10-04
|
||||
|
||||
Everything below this section is dated history; where it disagrees with this section, this section wins. The
|
||||
direction lives in `roadmap.md`, section of 2026-10-04 (phases G0-G5).
|
||||
|
||||
- **Goal.** A pre-push safety net that works on repositories it has never seen. Two numbers per push decide it:
|
||||
catch rate on real vulnerability-introducing changes, and false-alarm rate on ordinary changes. Neither has been
|
||||
measured yet; the unseen-repository pool (G0, spec `unseen-repo-evaluation`) comes first.
|
||||
- **Correction, 2026-10-04.** Infrastructure and engine work had drifted from that goal. It is now done only when
|
||||
it removes a bottleneck for those measurements.
|
||||
- **Engine.** Joern runs on the operator's kube-ax cluster as AX Tasks, with a VM fallback lane; kind stays the
|
||||
development profile. It is almost blind to fixes: paths differ between sides in 2 of 105 both-asked pairs.
|
||||
- **Decision engine.** Ranks but cannot block (exp-003 to exp-005 inconclusive; 710 fix pairs in memory). Fix
|
||||
mechanisms is one G2 arm, paused after mining until the pool exists.
|
||||
- **Hook.** `ousast pre-push` is operational and advisory on any repository; rollout stays NO-GO.
|
||||
|
||||
## State, 2026-10-02 (superseded by the section above)
|
||||
## Current state, 2026-10-02
|
||||
|
||||
Everything below this section is dated history; where it disagrees with this section, this section wins.
|
||||
User-facing documentation: `README.md`, `docs/architecture.md`, `docs/evaluation.md`, `docs/decision-engine.md`,
|
||||
`ops/ax/README.md`.
|
||||
|
||||
|
||||
@@ -1,143 +1,5 @@
|
||||
# Roadmap
|
||||
|
||||
## Roadmap to detection with the decision engine on kube-ax (2026-10-04)
|
||||
|
||||
This section supersedes the ordering below it. The older sections stay as history.
|
||||
|
||||
### Destination
|
||||
|
||||
On every push, from a hook or CI, OpenUltraSAST answers for the changed code. The answer is advisory for every
|
||||
family, and BLOCK only for families whose precision has been proven on repositories the system never saw. The work
|
||||
is split by what each place is good at:
|
||||
|
||||
- **kube-ax cluster** (operator k3s, Agent Substrate v0.3.0, AX v0.3.1, 2 nodes): the Joern engine, as AX Tasks.
|
||||
No API keys and no store credentials in the cluster; tasks read and write through short-lived presigned links.
|
||||
- **The VM or CI runner**: candidate selection, the decision engine's model calls (DeepSeek), the memory store,
|
||||
and the fallback engine lane (2.5 GB heap) for pins that do not fit the cluster's 1.7 GB heap.
|
||||
- **kind** stays the local development profile, with the same code and manifests.
|
||||
|
||||
### The question everything answers
|
||||
|
||||
Does the safety net work on repositories it has never seen? That is the purpose of the A/B experiments, and every
|
||||
phase below is judged by it. On an unseen repository almost every push is an ordinary change, so two numbers decide
|
||||
whether the safety net is useful, measured on the hook's advisory output exactly as a user would see it:
|
||||
|
||||
1. **Catch rate:** on real vulnerability-introducing changes in unseen repositories, how often it flags the change.
|
||||
2. **False-alarm rate:** on ordinary changes in the same unseen repositories, how often it flags anything.
|
||||
|
||||
Within-pair AUC on known fix pairs answers neither. It stays a development signal, not the goal.
|
||||
|
||||
### Where we are (2026-10-04)
|
||||
|
||||
| Area | State | Evidence |
|
||||
| --- | --- | --- |
|
||||
| Generalisation measured so far | The classifier ranks above chance on repository-held-out folds (within-pair AUC about 0.76) but cannot reach the block bar; the engine-based scan failed fresh populations (v1 1/11, v2 0/17); push-level catch and false-alarm rates on unseen repositories have never been measured | exp-003/004/005, `results-v1.json`, `results-v2.json` |
|
||||
| Pre-push hook | Operational and advisory on any repository | `2026-10-02-pre-push-operational-1/record.json` |
|
||||
| Labelled data | 710 complete real fix pairs | `2026-10-04-decision-engine-harvest-3/record.json` |
|
||||
| Engine | Almost blind to fixes (paths differ between sides in 2 of 105 both-asked pairs) | `2026-10-04-joern-slice-leak-audit/record.json` |
|
||||
| Infrastructure | Engine runs on kube-ax as AX Tasks with a VM fallback; kind for development | memory `kube-ax-cluster` |
|
||||
|
||||
Correction recorded 2026-10-04: the previous two days went mostly into infrastructure and engine internals that did
|
||||
not move a generalisation number. From here, infrastructure and engine work is done only when it removes a
|
||||
bottleneck for the measurements below.
|
||||
|
||||
### Where we are (2026-10-11)
|
||||
|
||||
The G0 measurement is now runnable: the unseen pool is **frozen** and the whole pipeline is hardened and
|
||||
**bucket-first**. The baseline replay is the one remaining step, to run on kube.
|
||||
|
||||
| Area | State | Evidence |
|
||||
| --- | --- | --- |
|
||||
| Unseen pool (G0.1) | **Frozen.** `pool-p1.toml` = 100 repos (slice 1); `freeze-p1.json` guard=0, dropped=0; deterministic (manifest `6bd2fa5d`). License 49 permissive / 51 private; methods 81 introducing / 19 last_touch. | `benchmarks/unseen/pool-p1.toml`; archived `ousast-corpus/unseen/20260601/pool/` |
|
||||
| Labels | 100 machine-derived (SZZ), **unchecked**. Record 07 is a **machine-label baseline** (caveat stated); a human label pass is a later refinement, before any published claim. | `freeze-p1.json` label_check_counts |
|
||||
| Harvest | Rebuilt to converge. Pure GitHub-API harvesting is a dead end for this benchmark (it needs the SZZ introducing commit + 12 stratified history negatives, not one commit). The fix: a full-objects clone (drop `--filter=blob:none`, which caused ~600 per-blob lazy fetches) + bucket-first durability + a get_repo cache that paid the 4,499-name used-set canonicalization once. Yield ~2.2%; **GitHub-rate-limit bound, not CPU**. | 95bed70, 857263d, 21aebe8, 7b711a6 |
|
||||
| Persistence | **Bucket-first, not local disk.** `ousast-corpus` is the source of truth: enumeration, resolution, get_repo, and accept/reject decisions all stream to and resume from it. The host disk (~9-13 GB free) is the scarce resource; RustFS/S3 has the room, so run artifacts and **replay outputs go to RustFS too**, not `~/ousast-results` on local disk. | `storage-streaming.md`, `bank.py` |
|
||||
| Replay image | Provenance pinned. CI `engine-image.yml` builds `ousast-replay-task` (harness with `/app/benchmarks/unseen/replay_entry.py`) + `ousast-engine-task` (Joern hook), publicly pullable by digest. The old `ousast-runner:dev` (248 MB ax-runner, no `/app/benchmarks`, no Joern) is the **wrong** image for replay. | CI run 38116281676 |
|
||||
| Baseline (G0.2) | **Not yet.** Replay reads the frozen pool cleanly (dry-run = 1,300 changes, `freeze_digest` verified). Remaining: mirror CI's passing `replay` job wiring, and stand up the kube lane (`ax` binary + a public-HTTPS presign-capable store endpoint; GHCR pull already public). | CI `replay` job (passed) |
|
||||
|
||||
Lesson, recorded so it is not re-paid: sourcing and engine-image work cost real time. The durable takeaway now lives
|
||||
in the pipeline and these docs — bucket-first persistence, pinned image provenance, a measured yield — so the next
|
||||
pool or arm is cheap, not a rediscovery.
|
||||
|
||||
### Readiness plan (to G0 exit = record 07)
|
||||
|
||||
1. **Kube replay lane (infra, operator-owned):** install the `ax` binary; provide the store endpoint the AX lane
|
||||
needs. The only thing settled in-repo is a code fact: `benchmarks/ax/batch.py` currently expects a public-HTTPS
|
||||
DNS, presign-capable S3 endpoint. How that endpoint and the cluster's access to the store are provided is an
|
||||
operator/AX decision, not an assumption to make here. GHCR pull of the images is verified public.
|
||||
2. **Unblock the replay** by mirroring CI's passing `replay` job (replay-task + engine-task wired, store + network).
|
||||
3. **Run slice-1 baseline on kube:** `--lane ax --arm default`, 1,300 changes, no model cost, resumable; outputs to
|
||||
RustFS.
|
||||
4. **Adjudicate (7.3) -> record 07 (7.4)** = the measured machine-label unseen baseline = **G0 exit**.
|
||||
5. Then **G1 arms** (DeepSeek decision engine) on the frozen pool.
|
||||
|
||||
### Phases
|
||||
|
||||
**G0. The unseen-repository pool and the push-level metric** (new spec `unseen-repo-evaluation`)
|
||||
|
||||
1. A frozen pool of repositories never used for training, tuning, mining or harvesting, and not in any population.
|
||||
Two kinds of change per repository: real vulnerability-introducing commits (the parent of a fix, post-cutoff where
|
||||
possible) and ordinary commits from the same history.
|
||||
2. The metric: replay each change through `ousast pre-push` (and the decision engine where an arm uses it) and
|
||||
record catch rate and false-alarm rate, with confidence intervals, per family and overall.
|
||||
3. Rotation: the pool is spent per decision, like the populations; a new slice is drawn for the next one.
|
||||
|
||||
Exit: the pool is frozen and the current production hook has a measured baseline on it.
|
||||
|
||||
**G1. Re-score the arms we have on the pool**
|
||||
|
||||
Current hook (quick rules + engine), decision-engine advisory (current programs), combiner, engine evidence. Each is
|
||||
an arm with a registered decision rule, scored by catch rate at a fixed false-alarm budget.
|
||||
|
||||
Exit: one measured table of arms on unseen code; keep only what generalises.
|
||||
|
||||
**G2. Improve what moves the unseen-repo numbers**
|
||||
|
||||
Candidates, each admitted only if it improves the pool metric on a fresh slice: fix mechanisms in the engine
|
||||
(approved plan, now one arm, not the spine), engine coverage and TypeScript, decision-engine prompt and retrieval
|
||||
changes, more labelled pairs from more distinct repositories.
|
||||
|
||||
Active G2 workstreams (2026-10-11, opened while driving G0; progress tracked in their briefs):
|
||||
|
||||
- **Change-scoped engine analysis** (`.kiro/specs/engine-change-scoped-analysis/brief.md`). Live diagnosis on the
|
||||
frozen pool showed the engine's "no finding" on unseen changes is usually *analysis-never-runs*: the engine ranks
|
||||
repo-wide candidates and `deferred: deadline_exhausted` before analyzing anything (even at 300 s), and the ranked
|
||||
sites are unrelated to the pushed change. Lever: seed analysis from the change's declarations + dataflow
|
||||
neighbourhood (`rank.py`/`mapping.py`) so it runs within budget. Increment 0 (confirm CPG-build-vs-analysis cost
|
||||
split) gates the rest. Scale measurement needs the kube lane.
|
||||
- **Line-correspondence (done, necessary-not-sufficient).** `snapshot.py` discarded base↔head anchors on
|
||||
non-globally-unique lines → `comparison_unknown` → suppressed findings; fixed to run-aware anchoring
|
||||
(commit c39fbb8, 163 tests). Removes one cause; coverage still needs change-scoped analysis + the capability
|
||||
registry (Task 8.3).
|
||||
|
||||
**G3. Decide BLOCK per family** on the pool's false-alarm rate and the pooled precision bound (Req 6.4).
|
||||
|
||||
**G4. Detection on push for any repository** (`plane-on-kubernetes` P4 work: `ousast plane scan --base --head`,
|
||||
engine pins on kube-ax, model calls on the VM or CI), sized by what G1-G3 kept.
|
||||
|
||||
**G5. Qualification and rollout:** adopt, commit `prediction-v3.json`, spend population v3 once, advisory rollout,
|
||||
BLOCK only for families that passed.
|
||||
|
||||
### Cross-cutting rules
|
||||
|
||||
- No hand-tuning per miss. Every detection change is a registered experiment or an admission by held-out
|
||||
separation.
|
||||
- Records carry counts, digests and pattern names, never repository names: the reserved-repository guard covers
|
||||
every population.
|
||||
- Instrument first: a zero from an unread input is a failed run (AGENTS.md).
|
||||
- Cluster stability: sandbox deaths are retried once, then sent to the VM lane; the operator owns workers, egress
|
||||
and Substrate.
|
||||
|
||||
### Spec map
|
||||
|
||||
| Spec | Role in this roadmap | State |
|
||||
| --- | --- | --- |
|
||||
| `unseen-repo-evaluation` (new) | G0, G1: the pool and the push-level metric | To specify |
|
||||
| `learned-decision-engine` | G1-G3, G5; fix-mechanisms plan is one G2 arm | Implementation; experiments 3-5 recorded |
|
||||
| `plane-on-kubernetes` | G4; the engine executors for kube-ax | Groups 1-2 done (kind); production path re-scoped to the operator's kube-ax (AX Tasks), groups 3-8 to re-plan |
|
||||
| `pre-push-safety-net` | the hook replayed in G0-G1; M4-M5 in G5 | Operational increment 1 done; M4-M5 open |
|
||||
| `ai-service-plane` | The plane the above runs on | Complete |
|
||||
|
||||
## Status, 2026-10-02
|
||||
|
||||
The milestone table below dates from 2026-09-14. Since then (`.kiro/specs/pre-push-safety-net/release-milestones.md`):
|
||||
|
||||
@@ -142,19 +142,3 @@ promising but does not qualify security queries or hook latency. The contributor
|
||||
vendor/correspondence completeness vetoes, and query-to-operation identity as the next
|
||||
shared-contract work. Resolving a boundary requires evidence of its relevance and scope;
|
||||
do not simply drop completeness checks to increase admissions. Rollout remains NO-GO.
|
||||
|
||||
On 2026-10-07, the **search-with-proof** arm reached its first milestone: an autonomous,
|
||||
execution-backed `demonstrated` finding end to end on the kube-ax cluster. On a real CVE
|
||||
fix-pair (hanxi/xiaomusic, path traversal of the prefix-check-bypass class), the agent
|
||||
found the vulnerability, authored a declarative demo, and the differential verifier
|
||||
confirmed the owned effect on the vulnerable revision and its absence on the fixed one;
|
||||
a blind search of the fixed revision produced zero false proofs. This is the proof-based
|
||||
BLOCK mechanism the maintainer directed after classification hit a precision wall: a BLOCK
|
||||
rests on a demonstrated finding with a direct execution trace, not a confidence score.
|
||||
Scope note: this is feasibility-pilot evidence on a development-corpus pair (known location,
|
||||
labelled family, runnable-project selection bias), not release readiness. The pilot exit
|
||||
(≥5 demonstrations in one family with 0 false proofs, Req 6.2) and the ≥73-negative BLOCK
|
||||
qualification (Req 6.3) remain open, and no capability is enabled. The decision/classification
|
||||
engine is retained as a triage ranker (it could not prove 95% precision as a gatekeeper);
|
||||
proof is the BLOCK gate. See `search-with-proof/HANDOFF.md` and
|
||||
`benchmarks/measurements/2026-10-07-search-first-cluster-demonstration/`.
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
# Stream run artifacts to RustFS, one bucket per purpose
|
||||
|
||||
Recorded 2026-10-09 from the maintainer's direction. The host's local disk is the scarce
|
||||
resource (it runs near full, and the draw stops itself at under 2 GB free). Runs should stream
|
||||
their durable artifacts to RustFS instead of accumulating them on local disk, and should spread
|
||||
them across **purpose-specific buckets** rather than one shared bucket. This relieves the disk and
|
||||
simplifies runs: local state becomes mostly a scratch cache that can be rebuilt or resumed from the
|
||||
object store.
|
||||
|
||||
## Why per-purpose buckets
|
||||
|
||||
One shared bucket (`sast-memory`) already bit us: an append-rewritten key on a versioned bucket
|
||||
with no noncurrent-version expiry ballooned to >10 GB (see the `engine-queue` command-upload fix and
|
||||
the lifecycle rules added 2026-10-09). Separate buckets let each kind of artifact carry the
|
||||
lifecycle it actually needs, and keep a transient firehose from polluting durable data.
|
||||
|
||||
Proposed split (names illustrative; the admin creates them and grants the store key):
|
||||
|
||||
- `sast-memory` — the plane memory store (existing): facts, excerpts, embeddings, index. Durable,
|
||||
versioned, with the noncurrent-version lifecycle now in place.
|
||||
- `sast-pools` — frozen unseen pools and their provenance: `pool-pN.toml`, `freeze-pN.json`, and the
|
||||
draw journal needed to resume/reproduce. Durable, versioned. This is also the "don't redo the
|
||||
sourcing" archive.
|
||||
- `sast-runs` (or `sast-journals`) — ephemeral run and queue state (engine-queue, replay results,
|
||||
per-run journals). Aggressive lifecycle: short current expiry + 1-day noncurrent expiry, since it
|
||||
is a transient firehose.
|
||||
- optionally `sast-traces` — engine/replay traces, if kept.
|
||||
|
||||
## Principles
|
||||
|
||||
- **Stream, don't hoard.** Write a run's durable outputs to its bucket as they are produced; keep
|
||||
only a working scratch (e.g. the live git clone, which cannot be streamed) on local disk, and let
|
||||
the disk guard stay as the backstop.
|
||||
- **Lifecycle per bucket**, tuned to the artifact: durable+versioned for pools/memory;
|
||||
short+noncurrent-expiry for transient run/queue state. Never a versioned bucket without a
|
||||
noncurrent-version rule.
|
||||
- **Resume from the object store**, so a wiped or small local disk does not lose a long run.
|
||||
- **Least privilege per bucket.** A run gets a key scoped to the bucket it writes; the store never
|
||||
creates or configures a bucket (admin does that once), consistent with the current S3Store.
|
||||
- Content discipline unchanged: no secrets, no exploit payloads; pool manifests keep the existing
|
||||
public/private (licence) split.
|
||||
|
||||
## Foundation and dependency
|
||||
|
||||
`src/openultrasast/plane/memory.py` already speaks boto3 to RustFS (S3Store) with startup
|
||||
verification of versioning/lifecycle/Select/tags; this direction extends that pattern to more
|
||||
buckets rather than inventing a new transport. New buckets and their grants are an **operator
|
||||
admin** action — the store credentials cannot create buckets or set lifecycle (confirmed by the
|
||||
`AccessDenied` on the 2026-10-09 lifecycle apply).
|
||||
|
||||
## Default: RustFS over local disk
|
||||
|
||||
Local disk on the VM is the scarce resource (it runs near full; the draw stops itself under 2 GB free).
|
||||
RustFS/S3 has far more room. So the default for any run's durable state and large artifacts is the bucket,
|
||||
not local disk. Local files are a rebuildable scratch cache only (the live git clone, a within-process
|
||||
journal). A long run must be resumable purely from the bucket, and its outputs — including replay result
|
||||
records — are written to RustFS, not `~/ousast-results` on local disk.
|
||||
|
||||
## Status (implemented 2026-10-10/11)
|
||||
|
||||
Bucket-first persistence is live, not a future spec. The unseen draw streams every stage to the
|
||||
`ousast-corpus` bucket (`http://10.0.0.54:9001`, creds in `.env` as `CORPUS_S3_*`; boto3 needs path-style +
|
||||
SigV4) and resumes from it:
|
||||
|
||||
- `unseen/<seed>/decisions/<sha256(repo+fix)>.json` — accept/reject per repository (idempotent; content dedup).
|
||||
- `unseen/<seed>/resolved/<sha256(ghsa)>.json` — per-advisory resolution.
|
||||
- `unseen/<seed>/advisories/list.json` — the enumerated advisory list.
|
||||
- `unseen/repos/<sha256(name)>.json` — get_repo cache, **not** seed-scoped (shared across seeds/runs), so the
|
||||
thousands of used-set canonicalization lookups are paid once ever.
|
||||
- `unseen/<seed>/pool/` — the frozen pool and its provenance (`pool-p1.toml`, `private-pool-p1.toml`,
|
||||
`freeze-p1.json`, `bisect-p1.json`, `protocol-p1.md`): the durable "don't redo the sourcing" archive.
|
||||
|
||||
Implementation: `benchmarks/unseen/bank.py` (`CorpusBank`, `corpus_settings`). The old local journal is a
|
||||
within-process cursor only; the bucket is the cross-run source of truth, so a restart (even after a code
|
||||
change) resumes cheaply. Still to do and consistent with the default above: replay result records
|
||||
(`replay.py --out`) currently default to `~/ousast-results/unseen/...` on local disk — point them at RustFS
|
||||
for the slice-1 baseline and after. The AX (kube) lane already uses presigned object transfer, which is the
|
||||
same principle. One in-repo code fact: `benchmarks/ax/batch.py` currently expects a public-HTTPS DNS,
|
||||
presign-capable endpoint. How the store endpoint and the cluster's access to it are provided is operator/AX-owned
|
||||
infra, not something this doc should presume.
|
||||
+42
-19
@@ -1,14 +1,26 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||
ARG TASK_BASE_TAG=base-unpublished
|
||||
ARG TASK_BASE_DIGEST
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
# OpenUltraSAST with its CPG engine, in one image.
|
||||
#
|
||||
# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both
|
||||
# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call,
|
||||
# no docker socket, no path translation between host and container, and no second code path in the backend to
|
||||
# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other
|
||||
# container, which means mounting the docker socket, and that is a large amount of privilege for a linter.
|
||||
#
|
||||
# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a
|
||||
# contributor a working tool from `docker compose run` with nothing installed on their machine.
|
||||
|
||||
FROM eclipse-temurin:21-jre-noble
|
||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||
|
||||
ARG JOERN_VERSION=v4.0.625
|
||||
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
||||
|
||||
# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a
|
||||
# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl unzip \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN set -eux; \
|
||||
@@ -20,28 +32,39 @@ RUN set -eux; \
|
||||
unzip -q "${ARCHIVE}" -d /opt; \
|
||||
rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \
|
||||
/opt/joern-cli/joern-parse --help > /dev/null
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
|
||||
# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses
|
||||
# the same extensionless command convention as the other frontends.
|
||||
RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \
|
||||
ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \
|
||||
fi
|
||||
|
||||
COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
||||
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
||||
&& rm /tmp/frontend-retention-install.py
|
||||
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
|
||||
# Create runtime identity before the application layers.
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src ./src
|
||||
COPY benchmarks ./benchmarks
|
||||
|
||||
# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate
|
||||
# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in.
|
||||
RUN python3 -m venv /venv \
|
||||
&& /venv/bin/pip install --no-cache-dir -e ".[semantic]"
|
||||
ENV PATH="/venv/bin:${PATH}"
|
||||
|
||||
# Analysis never needs root.
|
||||
# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than
|
||||
# failing installation or creating a second account with the same UID.
|
||||
RUN if getent passwd 1000 > /dev/null; then \
|
||||
usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \
|
||||
else useradd --create-home --uid 1000 ousast; fi
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||
COPY --chown=1000 benchmarks ./benchmarks
|
||||
RUN chown -R ousast /app
|
||||
else \
|
||||
useradd --create-home --uid 1000 ousast; \
|
||||
fi \
|
||||
&& chown -R ousast /app
|
||||
USER ousast
|
||||
|
||||
ENTRYPOINT ["python", "-m", "openultrasast.cli"]
|
||||
CMD ["--help"]
|
||||
|
||||
@@ -14,11 +14,6 @@ A scan combines up to four parts:
|
||||
Every finding carries the evidence that earned its place. When a scan could not look at
|
||||
something, it records a degradation. It does not report a clean result.
|
||||
|
||||
Where a repository's application can be built and run, an experimental proof arm goes further. It
|
||||
demonstrates a finding by executing it: the issue triggers on the vulnerable revision and not on
|
||||
the fixed one, so a demonstrated finding is a reproducible fact, not a score. This arm is in pilot
|
||||
and being qualified on repositories the tool has never seen; see [Status](#status).
|
||||
|
||||
`ousast` is the interface. Model work over whole repositories runs on a separate agentic plane
|
||||
(google/ax). A learned decision engine is in development. A scan needs neither of them.
|
||||
|
||||
@@ -28,30 +23,17 @@ The site is published at <https://norandom.github.io/OpenUltraSAST/>. Its source
|
||||
|
||||
## Status
|
||||
|
||||
**v2.0.1, state as of 2026-10-08.** The pre-push safety net is **not yet released**. Blocking on
|
||||
the static analyser by hand is **NO-GO** against every M4 gate, and the project no longer tries to
|
||||
reach them that way.
|
||||
**v2.0.0, state as of 2026-10-02.** The pre-push safety net is **NO-GO** against every M4 gate.
|
||||
|
||||
- No blocking hook capability is enabled; the pre-push hook is advisory only.
|
||||
- Two independent populations (case sets frozen before any scan) have been spent on the static
|
||||
engine: recall was 1 of 11, then 0 of 17. The learned decision engine ranks but blocks for no
|
||||
family.
|
||||
- Why blocking is hard: static abstract interpretation could not prove the precision a block
|
||||
needs. So the current work is a **dynamic proof arm**. It builds and runs a repository's app and
|
||||
demonstrates a finding by a differential: the issue triggers on the vulnerable revision and not
|
||||
on the fixed one. A demonstrated finding is ground truth by execution, so it can earn a block
|
||||
without calibration. Pilot demonstrations exist, including one executed on the cluster; the arm
|
||||
is being qualified, not shipped.
|
||||
- The release gate is a **measured** catch-rate and false-alarm number on repositories the tool
|
||||
has never seen (the unseen-repo evaluation), produced rather than asserted. Until that number
|
||||
exists, treat all output as advisory.
|
||||
- No hook capability is enabled.
|
||||
- Two independent populations (case sets frozen before any scan) have been spent so far. Engine
|
||||
recall was 1 of 11 and then 0 of 17.
|
||||
- The learned decision engine ranks but may not block. BLOCK is offered for no family.
|
||||
|
||||
Every detection change has to win a controlled measurement on held-out repositories. One page
|
||||
states where things stand, with the record behind each figure:
|
||||
[docs/where-we-stand.md](docs/where-we-stand.md). For developers, the pre-push check and its
|
||||
honest limits: [docs/pre-push-for-developers.md](docs/pre-push-for-developers.md); the
|
||||
catch–prove–fix–re-verify loop: [docs/use-case-fix-and-reverify.md](docs/use-case-fix-and-reverify.md).
|
||||
The release history is in [RELEASE_NOTES.md](RELEASE_NOTES.md).
|
||||
The project has stopped trying to reach the gates by hand. Every detection change now has to win
|
||||
a controlled measurement on held-out repositories. One page states all of this, with the record
|
||||
behind each figure: [docs/where-we-stand.md](docs/where-we-stand.md). The release history is in
|
||||
[RELEASE_NOTES.md](RELEASE_NOTES.md).
|
||||
|
||||
## Install and first scan
|
||||
|
||||
@@ -381,81 +363,3 @@ own terms:
|
||||
- Joern (Apache-2.0) is downloaded at image build time.
|
||||
- The tree-sitter wheels of the `semantic` extra are MIT.
|
||||
- The published benchmark cases were licence-checked one by one. v1's OpenCVE case is BUSL-1.1, see [RELEASE_NOTES.md](RELEASE_NOTES.md).
|
||||
|
||||
## Engine traces on kube-ax
|
||||
|
||||
`benchmarks/learn/engine_trace.py --executor ax` submits AX v0.3.1 Tasks with
|
||||
`ax apply -f -` and removes them with `ax delete task NAME`. The operator supplies
|
||||
kubeconfig and AX access; the dispatcher creates no Kubernetes resources or secrets.
|
||||
The `ousast-engine-task` GHCR package **must be made Public once** in its package
|
||||
settings so the actors can pull it without registry credentials.
|
||||
|
||||
The task image contains the current engine and worker code. Rebuild it after code
|
||||
changes and pass its digest with `--image`; no analyzer download is needed. The host
|
||||
materializes each pin as usual and uploads source and prepared questions under
|
||||
`engine-queue/<run>/<pin>/`. Only object-scoped presigned URLs and deadline parameters
|
||||
enter Task env. Credentials stay on the coordinator. Configure `OUSAST_MEMORY=s3://…`
|
||||
and the store credentials in the coordinator's `.env` or environment, with
|
||||
`S3_ENDPOINT=https://files.because-security.com`. Presigned links expire after the
|
||||
pin deadline plus ten minutes. IP and HTTP endpoints are rejected: the operator's
|
||||
actor EgressPolicy must allow **files.because-security.com:443**.
|
||||
|
||||
Completion is the result object, even while AX reports Running. Failed phases and
|
||||
host deadlines produce explicit failure/timeout rows. Task and staging-object cleanup
|
||||
runs on success and failure; cleanup failure stops further dispatch. Per-pin files,
|
||||
`--rerun-status`, resume and `progress.json` retain the existing trace-runner behavior.
|
||||
AX results include `executor: "ax"`, `worker_ip`, and the image digest. Placement is
|
||||
random; `--parallel` defaults to two AX tasks and cannot exceed two.
|
||||
|
||||
AX workers use `--heap-profile cluster` with
|
||||
`-Xms1700m -Xmx1700m -XX:MaxMetaspaceSize=256m -XX:ReservedCodeCacheSize=128m -XX:MaxDirectMemorySize=256m -Xss512k -XX:ActiveProcessorCount=1 -XX:+UseG1GC`
|
||||
on every frontend/query JVM; instruments record the flags. A `JVM out of memory`
|
||||
result is automatically re-queued to one local Docker lane with `--heap-profile vm`
|
||||
(2560m), recording `fallback: "vm-oom"` and the AX attempt. `mixed-ax` also uses this
|
||||
one Docker lane for overflow, prioritizing pending OOM fallbacks. Thus a mixed run has
|
||||
up to two AX tasks plus one Docker container. Use `--docker-image` for the locally
|
||||
available VM image; Docker never pulls it implicitly. A STOP file prevents new
|
||||
launches; interrupted fallback queues are recovered on resume.
|
||||
|
||||
Coordinator commands, from this worktree (the AX binary and kubeconfig remain
|
||||
configurable). Build/push the actual uncommitted tree and read its digest:
|
||||
|
||||
```bash
|
||||
cd /home/mc/Source/OpenUltraSAST/.claude/worktrees/engine-ax
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH="$PWD/src"
|
||||
TASK_TAG="ghcr.io/norandom/ousast-engine-task:sha-$(git rev-parse --short=7 HEAD)"
|
||||
docker buildx build --platform linux/amd64 -f plane/Dockerfile.engine-task \
|
||||
-t "$TASK_TAG" -t ghcr.io/norandom/ousast-engine-task:main \
|
||||
--push --metadata-file /tmp/ousast-engine-task-build.json .
|
||||
export TASK_IMAGE="ghcr.io/norandom/ousast-engine-task@$(python -c 'import json; print(json.load(open("/tmp/ousast-engine-task-build.json"))["containerimage.digest"])')"
|
||||
printf '%s\n' "$TASK_IMAGE"
|
||||
docker build -f Dockerfile -t openultrasast:engine-ax .
|
||||
```
|
||||
|
||||
The Engine image workflow also builds/pushes both images and prints each digest;
|
||||
CI needs the changes published before dispatch. The local build above includes
|
||||
uncommitted changes without publishing a git commit.
|
||||
|
||||
With `KUBECONFIG` pointing to the operator's kube-ax config and store settings in
|
||||
`.env`, run a one-pin smoke using the default experiment units and local example
|
||||
memory (or add `--examples /path/to/frozen-examples.jsonl`):
|
||||
|
||||
```bash
|
||||
: "${KUBECONFIG:?Set KUBECONFIG to the operator kube-ax configuration}"
|
||||
python benchmarks/learn/engine_trace.py --executor ax \
|
||||
--ax-bin "$HOME/.local/ax-v0.3.1/ax" --kubeconfig "$KUBECONFIG" \
|
||||
--atespace default --image "$TASK_IMAGE" --docker-image openultrasast:engine-ax \
|
||||
--parallel 1 --limit 1 --deadline 900 \
|
||||
--out "$HOME/ousast-results/plane/engine-ax-smoke"
|
||||
```
|
||||
|
||||
Then run the mixed lanes (repeat to resume; defaults re-run failed/timeout pins):
|
||||
|
||||
```bash
|
||||
python benchmarks/learn/engine_trace.py --executor mixed-ax \
|
||||
--ax-bin "$HOME/.local/ax-v0.3.1/ax" --kubeconfig "$KUBECONFIG" \
|
||||
--atespace default --image "$TASK_IMAGE" --docker-image openultrasast:engine-ax \
|
||||
--parallel 2 --deadline 900 --rerun-status failed,timeout \
|
||||
--out "$HOME/ousast-results/plane/engine-ax-mixed"
|
||||
```
|
||||
|
||||
@@ -1,809 +0,0 @@
|
||||
"""Host-only AX Task transport; credentials never enter task manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, deque
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
from openultrasast.config import load_dotenv
|
||||
from openultrasast.plane.memory import PRESIGNED_ARCHIVE_SUFFIX, S3Store, open_store
|
||||
from openultrasast.search.task_storage import diagnostic, exception_reason
|
||||
|
||||
|
||||
class EgressError(ValueError):
|
||||
"""Safe diagnostic without presigned URL contents."""
|
||||
|
||||
|
||||
class SandboxFailure(Exception):
|
||||
"""A terminal AX task without an authoritative result object."""
|
||||
|
||||
|
||||
class NoFreeWorkers(Exception):
|
||||
"""Capacity back-pressure, never a sandbox or instrument failure."""
|
||||
|
||||
|
||||
class ClusterUnavailable(Exception):
|
||||
"""No running workers were observed in the worker namespace."""
|
||||
|
||||
|
||||
class CapacityDeadline(Exception):
|
||||
"""No worker became available before the task deadline."""
|
||||
|
||||
|
||||
# Shared by every workload/lane instance in this dispatcher process, including
|
||||
# long-lived search executors. The operator runs one task per worker, two workers.
|
||||
AX_SLOTS = threading.BoundedSemaphore(2)
|
||||
|
||||
|
||||
def apply_when_available(lane, document, end):
|
||||
started = lane.clock()
|
||||
next_health_check = started + 60
|
||||
while lane.clock() < end:
|
||||
try:
|
||||
lane.ax("apply", "-f", "-", manifest=document, timeout=min(30, end - lane.clock()))
|
||||
return
|
||||
except NoFreeWorkers:
|
||||
# A failed apply is not proof that the server did not create a task.
|
||||
# Never send another create while the preceding task can still run.
|
||||
try:
|
||||
lane.ax("delete", "task", document["metadata"]["name"])
|
||||
except Exception as exc:
|
||||
raise RuntimeError("capacity cleanup failed: " + exception_reason(exc)) from None
|
||||
if lane.clock() >= next_health_check:
|
||||
lane.check_cluster_health(timeout=min(10, max(0.01, end - lane.clock())))
|
||||
next_health_check = lane.clock() + 60
|
||||
# Substrate's golden-snapshot copy holds the other worker for about a second after each new task
|
||||
# (operator, 2026-10-05): retry quickly for the first minute, then back off while real tasks run.
|
||||
delay = random.uniform(2, 5) if lane.clock() - started < 60 else random.uniform(15, 30)
|
||||
lane.pause(min(delay, max(0, end - lane.clock())))
|
||||
raise CapacityDeadline("no free workers available before deadline")
|
||||
|
||||
|
||||
def public_endpoint(url):
|
||||
"""Reject endpoints the actor's hostname-based HTTPS egress policy cannot reach."""
|
||||
parts = urlsplit(url)
|
||||
host = parts.hostname or ""
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
is_ip = True
|
||||
except ValueError:
|
||||
is_ip = False
|
||||
if (
|
||||
parts.scheme != "https"
|
||||
or not host
|
||||
or is_ip
|
||||
or "." not in host
|
||||
or host.endswith((".localhost", ".local", ".internal"))
|
||||
or parts.port not in (None, 443)
|
||||
or parts.username
|
||||
or parts.password
|
||||
):
|
||||
raise EgressError(
|
||||
"AX requires a public HTTPS DNS hostname in S3_ENDPOINT and presigned URLs: "
|
||||
"the egress policy allows the store hostname on port 443 only (no IP or plain HTTP)"
|
||||
)
|
||||
return host.lower()
|
||||
|
||||
|
||||
def task_state(text, name):
|
||||
"""Accept AX YAML/JSON output and its human-readable tasks table."""
|
||||
try:
|
||||
doc = yaml.safe_load(text)
|
||||
except yaml.YAMLError:
|
||||
doc = None
|
||||
rows = doc.get("items", doc.get("tasks", [doc])) if isinstance(doc, dict) else doc if isinstance(doc, list) else []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or row.get("metadata", {}).get("name", row.get("name")) != name:
|
||||
continue
|
||||
status = row.get("status", {})
|
||||
ip = next(
|
||||
(status.get(key) or row.get(key) for key in ("workerIP", "workerIp", "worker_ip", "ip") if status.get(key) or row.get(key)),
|
||||
None,
|
||||
)
|
||||
worker = status.get("worker")
|
||||
if not ip and isinstance(worker, dict):
|
||||
ip = worker.get("ip") or worker.get("address")
|
||||
return str(status.get("phase", row.get("phase", ""))), ip
|
||||
lines = [line.split() for line in text.splitlines() if line.strip()]
|
||||
if lines:
|
||||
headers = [re.sub(r"[^a-z]", "", cell.lower()) for cell in lines[0]]
|
||||
for cells in lines[1:]:
|
||||
if name not in cells:
|
||||
continue
|
||||
row = dict(zip(headers, cells, strict=False))
|
||||
phase = row.get("phase", row.get("status", ""))
|
||||
ip = row.get("workerip") or row.get("ip")
|
||||
if not ip:
|
||||
for cell in cells:
|
||||
try:
|
||||
ipaddress.ip_address(cell)
|
||||
ip = cell
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
return phase, None if ip in {"<none>", "-", ""} else ip
|
||||
listed = isinstance(doc, (dict, list)) or bool(lines and "name" in headers and {"phase", "status"}.intersection(headers))
|
||||
return "Missing" if listed else "", None
|
||||
|
||||
|
||||
def bounded_read(read, timeout):
|
||||
"""Do not let SDK retries postpone task teardown beyond the pin deadline.
|
||||
|
||||
Only the read runs in a daemon thread; it cannot publish a late result to disk.
|
||||
No executor shutdown waits for an SDK call whose socket has stopped responding.
|
||||
"""
|
||||
if timeout <= 0:
|
||||
raise TimeoutError("AX result deadline")
|
||||
replies = queue.Queue(maxsize=1)
|
||||
|
||||
def fetch():
|
||||
try:
|
||||
replies.put((True, read()))
|
||||
except Exception as exc:
|
||||
replies.put((False, exc))
|
||||
|
||||
threading.Thread(target=fetch, daemon=True, name="engine-ax-result-read").start()
|
||||
try:
|
||||
ok, result = replies.get(timeout=timeout)
|
||||
except queue.Empty:
|
||||
raise TimeoutError("AX result deadline") from None
|
||||
if not ok:
|
||||
raise result
|
||||
return result
|
||||
|
||||
|
||||
class AXLane:
|
||||
def __init__(self, args, workload, store=None, pause=time.sleep, clock=time.monotonic, runner=None):
|
||||
self.runner = runner or subprocess.run
|
||||
self.workload = workload
|
||||
self.args = args
|
||||
self.pause, self.clock = pause, clock
|
||||
self.run = uuid.uuid4().hex
|
||||
load_dotenv()
|
||||
self.host = public_endpoint(os.environ.get("S3_ENDPOINT", ""))
|
||||
# Validate the image before opening a store, whose constructor probes the bucket.
|
||||
validate_image(workload.image)
|
||||
self.store = store if store is not None else open_store()
|
||||
if store is None and not isinstance(self.store, S3Store):
|
||||
raise ValueError("AX requires OUSAST_MEMORY=s3://bucket[/prefix]")
|
||||
|
||||
def __call__(self, item, output, attempt):
|
||||
attempts = []
|
||||
# Docker has its own capacity and must not hold an AX worker slot.
|
||||
slot = not isinstance(self, DockerLane)
|
||||
if slot and not AX_SLOTS.acquire(timeout=self.workload.deadline):
|
||||
return {"status": "capacity_timeout", "ax_attempts": attempts}
|
||||
try:
|
||||
done, worker = self._execute_attempt(item, output, attempt, attempts)
|
||||
except CapacityDeadline:
|
||||
return {"status": "capacity_timeout", "ax_attempts": attempts}
|
||||
except SandboxFailure:
|
||||
return {"status": "sandbox_failure", "ax_attempts": attempts}
|
||||
except RuntimeError:
|
||||
# Teardown did not prove the worker free; retain its reservation.
|
||||
slot = False
|
||||
raise
|
||||
finally:
|
||||
if slot:
|
||||
AX_SLOTS.release()
|
||||
if done is None or done.returncode:
|
||||
return {
|
||||
"status": "instrument_failure",
|
||||
"reason": diagnostic(done.stderr) if done is not None else "TimeoutError: task result deadline",
|
||||
"ax_attempts": attempts,
|
||||
}
|
||||
record = json.loads((output / "result.json").read_text())
|
||||
return {**record, "worker_ip": worker, "ax_attempts": attempts}
|
||||
|
||||
def check_cluster_health(self, *, timeout=10):
|
||||
env = dict(os.environ)
|
||||
if self.args.kubeconfig:
|
||||
env["KUBECONFIG"] = str(Path(self.args.kubeconfig).expanduser())
|
||||
done = self.runner(
|
||||
["kubectl", "-n", "ax-workers", "get", "pods", "-o", "json"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if done.returncode:
|
||||
raise RuntimeError(f"worker pod health check failed (exit {done.returncode})")
|
||||
try:
|
||||
pods = json.loads(done.stdout)["items"]
|
||||
if not isinstance(pods, list):
|
||||
raise ValueError("invalid pod list")
|
||||
phases = [pod["status"]["phase"] for pod in pods]
|
||||
except (ValueError, KeyError, TypeError):
|
||||
raise RuntimeError("worker pod health check returned invalid pod data") from None
|
||||
running = phases.count("Running")
|
||||
if not running:
|
||||
raise ClusterUnavailable(f"worker pods: total={len(pods)}, running=0, pending={phases.count('Pending')}")
|
||||
|
||||
def ax(self, *args, manifest=None, timeout=30):
|
||||
env = dict(os.environ)
|
||||
if self.args.kubeconfig:
|
||||
env["KUBECONFIG"] = str(Path(self.args.kubeconfig).expanduser())
|
||||
done = self.runner(
|
||||
[str(Path(self.args.ax_bin).expanduser()), *args, *(["--atespace", self.args.atespace] if args[0] != "apply" else [])],
|
||||
input=json.dumps(manifest) if manifest is not None else None,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=max(0.01, timeout),
|
||||
)
|
||||
if done.returncode:
|
||||
if args[0] == "apply" and "no free workers available" in (done.stdout + done.stderr).lower():
|
||||
raise NoFreeWorkers("no free workers available")
|
||||
if args[0] == "delete" and "notfound" in (done.stdout + done.stderr).lower().replace(" ", ""):
|
||||
return done
|
||||
# AX errors may echo manifest URLs; sanitize before recording.
|
||||
raise ValueError(f"ax {args[0]} failed (exit {done.returncode}): " + diagnostic(done.stderr))
|
||||
return done
|
||||
|
||||
def _execute_attempt(self, item, output, attempt, attempts):
|
||||
pin_id = item.id
|
||||
digest = hashlib.sha256(f"{self.run}/{pin_id}/attempt-{attempt}".encode()).hexdigest()[:12]
|
||||
kind = re.sub(r"[^a-z0-9]+", "-", self.workload.kind.lower()).strip("-") or "pin"
|
||||
# Names are capped at 49 bytes: 14 (prefix) + kind + 1 + slug + 1 + 12 (digest).
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", str(pin_id).lower()).strip("-")[: max(4, 21 - len(kind))]
|
||||
name = f"ousast-engine-{kind}-{slug or 'pin'}-{digest}"
|
||||
key_id = hashlib.sha256(str(pin_id).encode()).hexdigest()
|
||||
prefix = f"engine-queue/{self.run}/{key_id}/attempt-{attempt}/"
|
||||
keys = [prefix + item.object_names.get(key, key.lower() + PRESIGNED_ARCHIVE_SUFFIX) for key in item.inputs]
|
||||
keys.append(prefix + item.result_object)
|
||||
part_keys = []
|
||||
expires = timedelta(seconds=self.workload.deadline + 600)
|
||||
worker_ip = None
|
||||
submitted = False
|
||||
started = self.clock()
|
||||
phase = ""
|
||||
try:
|
||||
for env, key, data in zip(item.inputs, keys, item.inputs.values(), strict=False):
|
||||
if env == "VERIFY_INPUT_URL":
|
||||
import io
|
||||
|
||||
from openultrasast.search.verify_task import MAX_PART_BYTES, upload_parts
|
||||
|
||||
count = (len(data) + MAX_PART_BYTES - 1) // MAX_PART_BYTES
|
||||
destinations = [key + f".part-{index}" for index in range(count)]
|
||||
part_keys.extend(destinations)
|
||||
put_urls = [self.store.presign_put(part, expires) for part in destinations]
|
||||
get_urls = [self.store.presign_get(part, expires) for part in destinations]
|
||||
if any(public_endpoint(url) != self.host for url in put_urls + get_urls):
|
||||
raise EgressError("archive part hostname must match store egress policy")
|
||||
from openultrasast.search.executor import URLTransport
|
||||
|
||||
data = upload_parts(
|
||||
io.BytesIO(data), lambda url, block: URLTransport().put(url, block, self.workload.deadline), put_urls
|
||||
)
|
||||
data = json.loads(data)
|
||||
for part, url in zip(data["parts"], get_urls, strict=True):
|
||||
part["url"] = url
|
||||
data = json.dumps(data).encode()
|
||||
self.store._put(key, data)
|
||||
urls = {env: self.store.presign_get(key, expires) for env, key in zip(item.inputs, keys, strict=False)}
|
||||
urls["RESULT_URL"] = self.store.presign_put(keys[-1], expires)
|
||||
if any(public_endpoint(url) != self.host for url in urls.values()):
|
||||
raise EgressError("AX presign hostname must match S3_ENDPOINT for the store egress policy")
|
||||
document = manifest(name, self.workload, item, urls, self.args.atespace)
|
||||
end = self.clock() + self.workload.deadline
|
||||
# An apply timeout can still have created the task; cleanup must cover it.
|
||||
submitted = True
|
||||
try:
|
||||
apply_when_available(self, document, end)
|
||||
except CapacityDeadline:
|
||||
submitted = False # All applies were explicitly rejected.
|
||||
raise
|
||||
while self.clock() < end:
|
||||
result = bounded_read(lambda: self.store._get(keys[-1]), end - self.clock())
|
||||
if self.clock() >= end:
|
||||
return None, worker_ip
|
||||
phase = ""
|
||||
try:
|
||||
state = self.ax("get", "tasks", timeout=min(30, end - self.clock())).stdout
|
||||
phase, ip = task_state(state, name)
|
||||
worker_ip = ip or worker_ip
|
||||
except (ValueError, OSError, subprocess.SubprocessError):
|
||||
# The object is authoritative even when AX status is temporarily unavailable.
|
||||
pass
|
||||
if self.clock() >= end:
|
||||
return None, worker_ip
|
||||
if result is not None:
|
||||
self.workload.validate(result[0], output)
|
||||
return subprocess.CompletedProcess([], 0, "", ""), worker_ip
|
||||
if phase.lower() in {"failed", "missing"}:
|
||||
# The worker may have uploaded while we were reading AX status.
|
||||
result = bounded_read(lambda: self.store._get(keys[-1]), end - self.clock())
|
||||
if self.clock() >= end:
|
||||
return None, worker_ip
|
||||
if result is not None:
|
||||
self.workload.validate(result[0], output)
|
||||
return subprocess.CompletedProcess([], 0, "", ""), worker_ip
|
||||
raise SandboxFailure
|
||||
self.pause(min(2, max(0, end - self.clock())))
|
||||
return None, worker_ip
|
||||
except (subprocess.TimeoutExpired, TimeoutError) as exc:
|
||||
return subprocess.CompletedProcess([], 1, "", exception_reason(exc)), worker_ip
|
||||
except (SandboxFailure, CapacityDeadline):
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Never expose exception strings which may contain object-scoped bearer URLs.
|
||||
detail = exception_reason(exc)
|
||||
return subprocess.CompletedProcess([], 1, "", "AX transport failed: " + detail), worker_ip
|
||||
finally:
|
||||
attempts.append({"name": name, "phase": phase, "seconds": self.clock() - started})
|
||||
errors = []
|
||||
if submitted:
|
||||
try:
|
||||
self.ax("delete", "task", name)
|
||||
except Exception as exc:
|
||||
# A successful list already confirmed a disappeared task is gone.
|
||||
if phase != "Missing":
|
||||
errors.append("task: " + exception_reason(exc))
|
||||
for key in [*keys, *part_keys]:
|
||||
try:
|
||||
self.store._delete(key)
|
||||
except Exception as exc:
|
||||
errors.append("object: " + exception_reason(exc))
|
||||
if errors:
|
||||
raise RuntimeError("AX cleanup failed; stop dispatching: " + ", ".join(errors))
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
inputs: dict[str, bytes]
|
||||
result_object: str
|
||||
command: tuple[str, ...]
|
||||
extra_env: dict[str, str]
|
||||
input_digest: str
|
||||
object_names: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Workload:
|
||||
image: str
|
||||
url_env: frozenset[str]
|
||||
deadline: float
|
||||
validate: object
|
||||
# Task-name prefix after "ousast-engine-"; the operator's per-host egress rules match on it.
|
||||
kind: str = "pin"
|
||||
|
||||
|
||||
def validate_image(image):
|
||||
if not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", image):
|
||||
raise ValueError("AX --image must be pinned by sha256 digest")
|
||||
|
||||
|
||||
def manifest(name, workload, item, urls, atespace="default", *, validate_name=True):
|
||||
validate_image(workload.image)
|
||||
if validate_name and (not re.fullmatch(r"ousast-engine-[a-z0-9-]+", name) or len(name.encode()) > 49):
|
||||
raise ValueError("invalid AX task name")
|
||||
if set(urls) != workload.url_env:
|
||||
raise ValueError("unexpected URL environment names")
|
||||
# Fixed names and semantic values, never inherited environment or credentials.
|
||||
patterns = {
|
||||
"OUSAST_SCRATCH_BYTES": r"[1-9][0-9]*",
|
||||
"DEADLINE": r"[0-9]+(?:\.[0-9]+)?",
|
||||
"QUESTION_DEADLINE": r"[0-9]+(?:\.[0-9]+)?",
|
||||
"SEARCH_STEP": r"reason|explore|verify",
|
||||
"SEARCH_ID": r"[a-z0-9][a-z0-9-]{0,63}",
|
||||
"SEARCH_TASK_ID": r"[a-z0-9][a-z0-9-]{0,63}",
|
||||
}
|
||||
if any(key not in patterns or not re.fullmatch(patterns[key], str(value)) for key, value in item.extra_env.items()):
|
||||
raise ValueError("unexpected task environment")
|
||||
if set(item.extra_env) & set(urls):
|
||||
raise ValueError("overlapping task environment")
|
||||
for key, value in urls.items():
|
||||
if key == "PREPARED_PART_URLS":
|
||||
from openultrasast.search.verify_task import MAX_PARTS
|
||||
|
||||
parts = json.loads(value)
|
||||
if not isinstance(parts, list) or len(parts) != MAX_PARTS:
|
||||
raise ValueError("invalid prepared part URLs")
|
||||
for url in parts:
|
||||
public_endpoint(url)
|
||||
else:
|
||||
public_endpoint(value)
|
||||
env = [{"name": key, "value": str(value)} for key, value in {**urls, **item.extra_env}.items()]
|
||||
if len(json.dumps(env).encode()) >= 32768:
|
||||
raise ValueError("AX Task env exceeds the 32 KB limit")
|
||||
return {
|
||||
"apiVersion": "ax.io/v1alpha1",
|
||||
"kind": "Task",
|
||||
"metadata": {"name": name, "atespace": atespace},
|
||||
"spec": {"image": workload.image, "env": env, "command": list(item.command)},
|
||||
}
|
||||
|
||||
|
||||
def write_json(path, record):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps(record, sort_keys=True) + "\n")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def schedule(pending, lanes, out, execute, complete, *, fallbacks=(), overflow=True, progress=None):
|
||||
"""The sole progress writer; callbacks carry workload-specific records, never transport."""
|
||||
if sum(lane in {"ax", "kind"} for lane in lanes) > 2 or lanes.count("docker") > 1:
|
||||
raise ValueError("at most two AX lanes and one VM lane")
|
||||
pending, fallbacks = deque(pending), deque(fallbacks)
|
||||
progress = progress if progress is not None else {"pins_done": 0, "pins_total": len(pending) + len(fallbacks)}
|
||||
progress["lanes"] = dict(Counter(lanes))
|
||||
active = {}
|
||||
failures = False
|
||||
|
||||
def persist():
|
||||
progress.update(pins_in_flight=len(active), active_lanes=dict(Counter(lane for _, lane in active.values())))
|
||||
if any(lane in {"ax", "kind"} for lane in lanes):
|
||||
progress["fallbacks_pending"] = len(fallbacks)
|
||||
write_json(out / "progress.json", progress)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(lanes)) as pool:
|
||||
|
||||
def launch(lane):
|
||||
if (out / "STOP").exists():
|
||||
return
|
||||
if lane == "docker" and fallbacks:
|
||||
item, previous = fallbacks.popleft()
|
||||
elif pending and (lane != "docker" or overflow):
|
||||
item, previous = pending.popleft(), None
|
||||
else:
|
||||
return
|
||||
active[pool.submit(execute, item, lane, previous)] = item, lane
|
||||
|
||||
for lane in lanes:
|
||||
launch(lane)
|
||||
persist()
|
||||
try:
|
||||
while active:
|
||||
finished, _ = wait(active, return_when=FIRST_COMPLETED)
|
||||
# Resolve ALL completed futures before launching; cleanup exceptions stop launches.
|
||||
results = [(active.pop(future), future.result()) for future in finished]
|
||||
for (item, lane), result in results:
|
||||
failed, fallback = complete(item, lane, result, progress)
|
||||
if fallback is not None:
|
||||
fallbacks.append((item, fallback))
|
||||
else:
|
||||
failures |= failed
|
||||
progress["pins_done"] += 1
|
||||
for lane, count in (Counter(lanes) - Counter(lane for _, lane in active.values())).items():
|
||||
for _ in range(count):
|
||||
launch(lane)
|
||||
persist()
|
||||
finally:
|
||||
persist()
|
||||
return failures
|
||||
|
||||
|
||||
def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), overflow=True, max_attempts=2):
|
||||
"""Digest checkpoints, one instrument rerun, sandbox retry, priority VM fallback."""
|
||||
if not isinstance(max_attempts, int) or not 1 <= max_attempts <= 2:
|
||||
raise ValueError("batch attempts must be one or two")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
items = list(items)
|
||||
if len({item.id for item in items}) != len(items):
|
||||
raise ValueError("duplicate dispatch item id")
|
||||
pending, fallbacks = [], []
|
||||
|
||||
def path(item):
|
||||
return out / (hashlib.sha256(item.id.encode()).hexdigest() + ".json")
|
||||
|
||||
for item in items:
|
||||
previous = json.loads(path(item).read_text()) if path(item).exists() else {}
|
||||
if previous.get("input_digest") == item.input_digest:
|
||||
if previous.get("fallback_pending"):
|
||||
fallbacks.append((item, previous))
|
||||
continue
|
||||
if previous.get("done"):
|
||||
continue
|
||||
pending.append(item)
|
||||
|
||||
def execute(item, lane, previous):
|
||||
output = out / hashlib.sha256(item.id.encode()).hexdigest()
|
||||
output.mkdir(exist_ok=True)
|
||||
attempts = []
|
||||
for attempt in range(max_attempts):
|
||||
attempt_output = output / ("attempt-" + uuid.uuid4().hex)
|
||||
attempt_output.mkdir()
|
||||
record = executors[lane](item, attempt_output, attempt)
|
||||
for artifact in attempt_output.iterdir():
|
||||
if artifact.is_file():
|
||||
shutil.copyfile(artifact, output / artifact.name)
|
||||
attempts.append(
|
||||
{
|
||||
"lane": lane,
|
||||
"status": record.get("status"),
|
||||
"attempt": attempt,
|
||||
"transport": record.get("ax_attempts", []),
|
||||
"docker_errors": record.get("docker_errors", []),
|
||||
}
|
||||
)
|
||||
if record.get("status") not in {"sandbox_failure", "instrument_failure"}:
|
||||
break
|
||||
return {**record, "attempts": (previous or {}).get("attempts", []) + attempts, "input_digest": item.input_digest, "done": True}
|
||||
|
||||
def complete(item, lane, record, progress):
|
||||
fallback = lane in {"ax", "kind"} and record.get("status") in {"sandbox_failure", "oom"} and "docker" in lanes
|
||||
record["fallback_pending"] = fallback
|
||||
write_json(path(item), record)
|
||||
progress["last_item"] = hashlib.sha256(item.id.encode()).hexdigest()
|
||||
return record.get("status") not in {"ok", "unanalysable", "quick_only", "engine_covered"}, record if fallback else None
|
||||
|
||||
return schedule(pending, lanes, out, execute, complete, fallbacks=fallbacks, overflow=overflow)
|
||||
|
||||
|
||||
def docker_diagnostic(text):
|
||||
"""Bound diagnostics and remove bearer URLs, environment secrets and assignments."""
|
||||
return diagnostic(text or "", maximum=4000)
|
||||
|
||||
|
||||
class DockerLane(AXLane):
|
||||
"""VM lane uses the same presigned-object protocol and pinned replay image."""
|
||||
|
||||
def __init__(self, *args, runner=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.runner = runner or subprocess.run
|
||||
self._prepared = False
|
||||
|
||||
def __call__(self, item, output, attempt):
|
||||
self._diagnostics = []
|
||||
try:
|
||||
if not self._prepared:
|
||||
self._prepare_image()
|
||||
self._prepared = True
|
||||
except (ValueError, OSError, subprocess.SubprocessError) as exc:
|
||||
if not self._diagnostics:
|
||||
self._diagnostics.append({"step": "preflight", "exit_code": None, "stderr": docker_diagnostic(str(exc))})
|
||||
record = {"status": "instrument_failure", "docker_errors": self._diagnostics}
|
||||
write_json(output / "docker-error.json", record)
|
||||
return record
|
||||
try:
|
||||
record = super().__call__(item, output, attempt)
|
||||
finally:
|
||||
if self._diagnostics:
|
||||
write_json(output / "docker-error.json", {"docker_errors": self._diagnostics})
|
||||
if self._diagnostics:
|
||||
record["docker_errors"] = self._diagnostics
|
||||
return record
|
||||
|
||||
def _run_docker(self, command, timeout):
|
||||
try:
|
||||
done = self.runner(command, capture_output=True, text=True, timeout=timeout)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
self._diagnostics.append(
|
||||
{"step": command[1], "exit_code": None, "stderr": docker_diagnostic(getattr(exc, "stderr", "") or exception_reason(exc))}
|
||||
)
|
||||
raise
|
||||
if done.returncode:
|
||||
self._diagnostics.append({"step": command[1], "exit_code": done.returncode, "stderr": docker_diagnostic(done.stderr)})
|
||||
raise ValueError(f"docker {command[1]} failed (exit {done.returncode}): " + diagnostic(done.stderr))
|
||||
return done
|
||||
|
||||
def _prepare_image(self):
|
||||
# Docker's data root, not the coordinator checkout, is where layers consume space.
|
||||
root = self._run_docker(["docker", "info", "--format", "{{.DockerRootDir}}"], 30).stdout.strip()
|
||||
size = getattr(self.args, "docker_image_bytes", None)
|
||||
if not isinstance(size, int) or size <= 0:
|
||||
raise ValueError("set --docker-image-bytes to the unpacked image size upper bound before pulling")
|
||||
free = self.runner(["df", "-B1", "--output=avail", root], capture_output=True, text=True, timeout=30)
|
||||
if free.returncode:
|
||||
raise ValueError("cannot check free space on Docker data root")
|
||||
available = int(free.stdout.splitlines()[-1].strip())
|
||||
required = size + 1024**3
|
||||
if available < required:
|
||||
raise ValueError(f"refusing Docker pull: free bytes {available} below image + 1 GiB ({required})")
|
||||
self._run_docker(["docker", "pull", self.workload.image], 900)
|
||||
|
||||
def _execute_attempt(self, item, output, attempt, attempts):
|
||||
# Reuse all object/validation/cleanup logic, replacing only the task runner.
|
||||
import copy
|
||||
|
||||
lane = copy.copy(self)
|
||||
lane.ax = lane._docker_command
|
||||
lane._docker_name = None
|
||||
return AXLane._execute_attempt(lane, item, output, attempt, attempts)
|
||||
|
||||
def _docker_command(self, *args, manifest=None, timeout=30):
|
||||
if args[0] == "apply":
|
||||
self._docker_name = manifest["metadata"]["name"]
|
||||
self._docker_created = True # Even a failed run can leave a created container.
|
||||
command = [
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--runtime",
|
||||
"runc",
|
||||
"--pull",
|
||||
"never",
|
||||
"--name",
|
||||
self._docker_name,
|
||||
"--entrypoint",
|
||||
manifest["spec"]["command"][0],
|
||||
]
|
||||
for env in manifest["spec"]["env"]:
|
||||
command += ["-e", env["name"] + "=" + env["value"]]
|
||||
command += [manifest["spec"]["image"], *manifest["spec"]["command"][1:]]
|
||||
if not {"openultrasast.search.executor", "openultrasast.search.verify_task"}.intersection(manifest["spec"]["command"]):
|
||||
command += ["--heap-profile", "vm"]
|
||||
elif args[0] == "get":
|
||||
command = ["docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}}", self._docker_name]
|
||||
else:
|
||||
if not getattr(self, "_docker_created", False):
|
||||
return subprocess.CompletedProcess([], 0, "", "")
|
||||
command = ["docker", "rm", "-f", self._docker_name or args[2]]
|
||||
try:
|
||||
done = self._run_docker(command, timeout)
|
||||
except ValueError:
|
||||
if args[0] == "delete" and "No such container" in self._diagnostics[-1]["stderr"]:
|
||||
self._diagnostics.pop()
|
||||
return subprocess.CompletedProcess(command, 0, "", "")
|
||||
raise
|
||||
if args[0] == "apply":
|
||||
self._docker_created = True
|
||||
if args[0] == "get":
|
||||
phase = "Running" if done.stdout.startswith("true") else "Failed"
|
||||
if phase == "Failed":
|
||||
logs = self._run_docker(["docker", "logs", "--tail", "10", self._docker_name], timeout)
|
||||
self._diagnostics.append(
|
||||
{"step": "container", "exit_code": int(done.stdout.split()[-1]), "stderr": docker_diagnostic(logs.stderr)}
|
||||
)
|
||||
done.stdout = json.dumps({"metadata": {"name": self._docker_name}, "status": {"phase": phase}})
|
||||
return done
|
||||
|
||||
|
||||
class SearchExecutorTask:
|
||||
"""Host-owned executor lifecycle using the shared AX/presigning transport.
|
||||
|
||||
Supply a bounded tar archive of the checkout. The brain receives only the
|
||||
command PUT/result GET pair; the task receives their opposite permissions.
|
||||
Closing the context deletes the task even after an uncertain command.
|
||||
"""
|
||||
|
||||
def __init__(self, lane, checkout_archive, *, deadline=900):
|
||||
if not 0 < deadline <= 3600 or len(checkout_archive) > 128 * 1024**2:
|
||||
raise ValueError("executor task input limit")
|
||||
if isinstance(lane, DockerLane):
|
||||
import copy
|
||||
|
||||
lane = copy.copy(lane)
|
||||
lane._diagnostics = []
|
||||
lane.ax = lane._docker_command
|
||||
self.lane, self.archive, self.deadline = lane, checkout_archive, deadline
|
||||
token = uuid.uuid4().hex[:20]
|
||||
self.name = "ousast-engine-search-exec-" + token
|
||||
self.keys = [f"engine-queue/search-exec/{token}/{name}" for name in ("repo.tar", "command.json", "result.json", "prepared.json")]
|
||||
from openultrasast.search.verify_task import MAX_PARTS
|
||||
|
||||
self.keys.extend(self.keys[3] + f".part-{index}" for index in range(MAX_PARTS))
|
||||
self.submitted = False
|
||||
self.slot = False
|
||||
|
||||
def __enter__(self):
|
||||
from openultrasast.search.executor import ObjectStoreExecutor
|
||||
|
||||
self.end = time.monotonic() + self.deadline
|
||||
if not isinstance(self.lane, DockerLane):
|
||||
if not AX_SLOTS.acquire(timeout=self.deadline):
|
||||
raise CapacityDeadline("no free workers available before deadline")
|
||||
self.slot = True
|
||||
expires = timedelta(seconds=self.deadline + 600)
|
||||
store = self.lane.store
|
||||
try:
|
||||
if isinstance(self.lane, DockerLane):
|
||||
self.lane._run_docker(["docker", "image", "inspect", self.lane.workload.image], min(30, self.deadline))
|
||||
store._put(self.keys[0], self.archive)
|
||||
urls = {
|
||||
"REPO_URL": store.presign_get(self.keys[0], expires),
|
||||
"COMMAND_GET_URL": store.presign_get(self.keys[1], expires),
|
||||
"EXECUTOR_RESULT_PUT_URL": store.presign_put(self.keys[2], expires),
|
||||
"PREPARED_PUT_URL": store.presign_put(self.keys[3], expires),
|
||||
}
|
||||
part_urls = [store.presign_put(key, expires) for key in self.keys[4:]]
|
||||
command_put = store.presign_put(self.keys[1], expires)
|
||||
result_get = store.presign_get(self.keys[2], expires)
|
||||
if any(public_endpoint(url) != self.lane.host for url in [*urls.values(), *part_urls, command_put, result_get]):
|
||||
raise EgressError("executor presign hostname must match store egress policy")
|
||||
urls["PREPARED_PART_URLS"] = json.dumps(part_urls)
|
||||
work = Workload(self.lane.workload.image, frozenset(urls), self.deadline, None)
|
||||
item = Item(
|
||||
self.name,
|
||||
{},
|
||||
"result.json",
|
||||
(
|
||||
"python3",
|
||||
"-m",
|
||||
"openultrasast.search.executor",
|
||||
"--repo",
|
||||
"/workspace/checkout",
|
||||
"--task-boundary",
|
||||
"--deadline",
|
||||
str(self.deadline),
|
||||
),
|
||||
{},
|
||||
"",
|
||||
)
|
||||
document = manifest(self.name, work, item, urls, self.lane.args.atespace)
|
||||
self.submitted = True
|
||||
try:
|
||||
apply_when_available(self.lane, document, self.end)
|
||||
except CapacityDeadline:
|
||||
self.submitted = False
|
||||
raise
|
||||
self.client = ObjectStoreExecutor(command_put, result_get)
|
||||
return self
|
||||
except BaseException as exc:
|
||||
self.__exit__(type(exc), exc, exc.__traceback__)
|
||||
raise
|
||||
|
||||
def prepare(self, spec):
|
||||
"""Check the export manifest and every fragment before handing off the archive."""
|
||||
remaining = min(300, self.end - time.monotonic())
|
||||
result = self.client.submit("prepare_verification", spec, timeout_seconds=remaining)
|
||||
self.preparation_result = result
|
||||
if result.get("status") != "ok":
|
||||
from openultrasast.search.task_storage import CommandFailure
|
||||
|
||||
raise CommandFailure(
|
||||
result.get("phase", "build"), result.get("exit_code"), result.get("stderr", ""), result.get("reason", "could_not_build")
|
||||
)
|
||||
stored = bounded_read(lambda: self.lane.store._get(self.keys[3]), self.end - time.monotonic())
|
||||
if stored is None or hashlib.sha256(stored[0]).hexdigest() != result.get("sha256"):
|
||||
raise ValueError("executor archive missing or digest mismatch")
|
||||
import io
|
||||
|
||||
from openultrasast.search.verify_task import assemble_parts
|
||||
|
||||
def get(part):
|
||||
value = bounded_read(lambda: self.lane.store._get(self.keys[4 + part["index"]]), self.end - time.monotonic())
|
||||
if value is None:
|
||||
raise ValueError("executor archive part missing")
|
||||
return value[0]
|
||||
|
||||
archive = io.BytesIO()
|
||||
assemble_parts(stored[0], archive, get)
|
||||
return archive.getvalue()
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
errors = []
|
||||
if self.submitted:
|
||||
try:
|
||||
self.lane.ax("delete", "task", self.name)
|
||||
self.submitted = False
|
||||
except Exception as error:
|
||||
errors.append(exception_reason(error))
|
||||
for key in self.keys:
|
||||
try:
|
||||
self.lane.store._delete(key)
|
||||
except Exception as error:
|
||||
errors.append(exception_reason(error))
|
||||
if self.slot and not errors:
|
||||
AX_SLOTS.release()
|
||||
self.slot = False
|
||||
if errors:
|
||||
reason = "executor cleanup failed: " + ", ".join(errors)
|
||||
logging.getLogger(__name__).error("%s", reason)
|
||||
if exc is not None:
|
||||
exc.executor_cleanup_reason = reason
|
||||
exc.args = (str(exc) + "; " + reason,)
|
||||
return False
|
||||
raise RuntimeError(reason)
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Dispatch the packaged no-model probe using the shared engine/replay transport.
|
||||
|
||||
python -m benchmarks.ax.search_probe --lane ax --image IMAGE@sha256:DIGEST --out /tmp/probe
|
||||
For kind, select --lane kind and supply its --kubeconfig; Docker uses the same PUT protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import AXLane, DockerLane, Item, Workload, dispatch, validate_image
|
||||
|
||||
|
||||
def validate_result(data: bytes, output: Path) -> None:
|
||||
if len(data) > 1024 * 1024:
|
||||
raise ValueError("oversized probe result")
|
||||
record = json.loads(data)
|
||||
if not isinstance(record, dict) or record.get("status") not in ("ok", "instrument_failure"):
|
||||
raise ValueError("invalid probe result")
|
||||
if record["status"] == "ok" and (
|
||||
set(record.get("oracles", {})) != {"sql", "path", "command", "ssrf", "xss"}
|
||||
or record.get("isolation_check", {}).get("status") not in ("ok", "skipped")
|
||||
):
|
||||
raise ValueError("incomplete probe")
|
||||
(output / "result.json").write_text(json.dumps(record))
|
||||
|
||||
|
||||
def probe_workload(image: str, deadline: float = 900) -> Workload:
|
||||
validate_image(image)
|
||||
return Workload(image, frozenset({"RESULT_URL"}), deadline, validate_result, kind="search-probe")
|
||||
|
||||
|
||||
def probe_item(image: str) -> Item:
|
||||
return Item(
|
||||
"v1",
|
||||
{},
|
||||
"result.json",
|
||||
("python3", "-m", "openultrasast.search.probe"),
|
||||
{},
|
||||
hashlib.sha256(("search-probe-v1:" + image).encode()).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
def run_probe(args) -> bool:
|
||||
workload = probe_workload(args.image, args.deadline)
|
||||
transport = (DockerLane if args.lane == "docker" else AXLane)(args, workload)
|
||||
return dispatch([probe_item(args.image)], workload, args.out, {args.lane: transport}, lanes=(args.lane,), max_attempts=1)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--lane", choices=("ax", "kind", "docker"), default="ax")
|
||||
parser.add_argument("--out", required=True, type=Path)
|
||||
parser.add_argument("--deadline", type=float, default=900)
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--atespace", default="default")
|
||||
parser.add_argument("--kubeconfig")
|
||||
parser.add_argument("--docker-image-bytes", type=int)
|
||||
return 1 if run_probe(parser.parse_args()) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,124 +0,0 @@
|
||||
"""Dispatch three fresh key-free AX tasks for each verification side."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import AXLane, CapacityDeadline, Item, SearchExecutorTask, Workload, validate_image
|
||||
from openultrasast.search.budget import verification_run_seconds
|
||||
from openultrasast.search.verify import RunObservation, SideRecord
|
||||
from openultrasast.search.verify_task import MAX_RESULT_BYTES, pack_checkout
|
||||
|
||||
|
||||
def side_record(value):
|
||||
if not isinstance(value, dict) or value.get("isolation_mode") != "task-boundary":
|
||||
raise ValueError("missing task-boundary attestation")
|
||||
fields = SideRecord.__dataclass_fields__
|
||||
# AXLane adds transport metadata outside the task record.
|
||||
data = {key: value[key] for key in fields if key in value}
|
||||
if data.get("outcome") not in {"observed", "could_not_run", "could_not_build", "no_oracle", "inconclusive"}:
|
||||
raise ValueError("invalid side outcome")
|
||||
runs = data.get("runs")
|
||||
if not isinstance(runs, list) or len(runs) > 1:
|
||||
raise ValueError("task must return one repetition")
|
||||
for run in runs:
|
||||
if not isinstance(run, dict):
|
||||
raise ValueError("invalid observation")
|
||||
if type(run.get("observed")) is not bool or not isinstance(run.get("evidence"), str):
|
||||
raise ValueError("invalid observation")
|
||||
if data["outcome"] == "observed" and len(runs) != 1:
|
||||
raise ValueError("missing completed observation")
|
||||
for number in [
|
||||
data.get("elapsed_seconds"),
|
||||
*(data.get(k, 0) for k in ("build_seconds", "ready_seconds", "run_seconds", "scratch_peak_bytes")),
|
||||
*(run.get("elapsed_seconds") for run in runs),
|
||||
]:
|
||||
if type(number) not in (int, float) or not math.isfinite(number) or number < 0:
|
||||
raise ValueError("invalid observation measurement")
|
||||
if not isinstance(data.get("reason"), str):
|
||||
raise ValueError("invalid side reason")
|
||||
data["runs"] = tuple(RunObservation(**run) for run in runs)
|
||||
return SideRecord(**data)
|
||||
|
||||
|
||||
def validate_result(data, output):
|
||||
if len(data) > MAX_RESULT_BYTES:
|
||||
raise ValueError("oversized side result")
|
||||
side_record(json.loads(data))
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
(output / "result.json").write_bytes(data)
|
||||
|
||||
|
||||
class AXSideDispatcher:
|
||||
"""Host adapter passed as verify(task_dispatcher=...); never caches runs."""
|
||||
|
||||
def __init__(self, args, image, output, *, lane=None, prepare=None):
|
||||
validate_image(image)
|
||||
self.output = Path(output)
|
||||
self.args, self.image, self.lane = args, image, lane
|
||||
self.prepare = prepare
|
||||
|
||||
def __call__(self, *, side, demo, family, timeout_seconds, fresh_task):
|
||||
if fresh_task is not True or side.command or side.build_command or side.dependencies:
|
||||
raise ValueError("verification accepts checkout and declarative inputs only")
|
||||
# The trusted host selects an executor for this side. Build/package work
|
||||
# finishes there before any verifier worker slot is acquired.
|
||||
workload = Workload(
|
||||
self.image,
|
||||
frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}),
|
||||
verification_run_seconds(demo, timeout_seconds) + 60,
|
||||
validate_result,
|
||||
kind="search-verify",
|
||||
)
|
||||
lane = self.lane or AXLane(self.args, workload)
|
||||
spec = {"demo": demo, "family": family, "timeout_seconds": timeout_seconds}
|
||||
try:
|
||||
if self.prepare is not None:
|
||||
archive = self.prepare(side, spec)
|
||||
else:
|
||||
with SearchExecutorTask(lane, pack_checkout(side.checkout)) as executor:
|
||||
archive = executor.prepare(spec)
|
||||
except (ValueError, CapacityDeadline) as exc:
|
||||
return SideRecord("could_not_build", (), 0, str(exc), isolation_mode="task-boundary")
|
||||
records = []
|
||||
for _ in range(3):
|
||||
identity = uuid.uuid4().hex
|
||||
item = Item(
|
||||
identity,
|
||||
{"VERIFY_INPUT_URL": archive},
|
||||
"result.json",
|
||||
("python3", "-m", "openultrasast.search.verify_task"),
|
||||
{},
|
||||
hashlib.sha256(archive).hexdigest(),
|
||||
)
|
||||
output = self.output / identity
|
||||
output.mkdir(parents=True)
|
||||
raw = lane(item, output, 1)
|
||||
# Roundtrip enforces JSON types for test adapters as for remote tasks.
|
||||
if raw.get("status") == "capacity_timeout":
|
||||
return SideRecord("could_not_run", (), 0, "AX capacity deadline", isolation_mode="task-boundary")
|
||||
result = side_record(json.loads(json.dumps(raw)))
|
||||
records.append(result)
|
||||
if result.outcome != "observed":
|
||||
break
|
||||
last = records[-1]
|
||||
return SideRecord(
|
||||
last.outcome,
|
||||
tuple(run for r in records for run in r.runs),
|
||||
sum(r.elapsed_seconds for r in records),
|
||||
last.reason,
|
||||
sum(r.build_seconds for r in records),
|
||||
sum(r.ready_seconds for r in records),
|
||||
sum(r.run_seconds for r in records),
|
||||
max(r.scratch_peak_bytes for r in records),
|
||||
isolation_mode="task-boundary",
|
||||
chromium_no_sandbox=any(r.chromium_no_sandbox for r in records),
|
||||
phase=last.phase,
|
||||
exit_code=last.exit_code,
|
||||
stderr=last.stderr,
|
||||
stdout=last.stdout,
|
||||
)
|
||||
@@ -126,9 +126,8 @@ def resolve_scan(scans: Path, case: str, side: str) -> dict[str, Any] | None:
|
||||
|
||||
|
||||
def recorded_engine(language: str, findings: list, result: Mapping | None, repository: bool) -> Part:
|
||||
# The language list describes standalone scans. Repository scans also analyze
|
||||
# TypeScript; recorded function findings take precedence over that list.
|
||||
if language not in ENGINE_LANGUAGES and not findings and not (repository and language == "typescript" and result is not None):
|
||||
# Recorded findings take precedence over the shared engine language coverage.
|
||||
if language not in ENGINE_LANGUAGES and not findings:
|
||||
return NONE
|
||||
state = engine_for(result)
|
||||
if state is not None:
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
"""AX task entry: object-scoped transfers and stdlib only; result PUT is completion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TaskEntryError(RuntimeError):
|
||||
"""A task-entry diagnostic containing only locally controlled text."""
|
||||
|
||||
|
||||
def transfer(request, destination=None, *, readiness_budget=None):
|
||||
"""Wait for actor egress on startup; bound later transfers to three attempts."""
|
||||
deadline = time.monotonic() + readiness_budget if readiness_budget is not None else None
|
||||
delay = 0.5
|
||||
attempt = 0
|
||||
while True:
|
||||
remaining = deadline - time.monotonic() if deadline is not None else 60
|
||||
if remaining <= 0:
|
||||
raise TaskEntryError("egress not open after 60 s")
|
||||
attempt += 1
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=min(60, remaining)) as response:
|
||||
if destination is not None:
|
||||
with destination.open("wb") as output:
|
||||
shutil.copyfileobj(response, output)
|
||||
return
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.close()
|
||||
if exc.code != 403 and not 500 <= exc.code < 600:
|
||||
raise
|
||||
if deadline is None and attempt >= 3:
|
||||
raise
|
||||
except (urllib.error.URLError, OSError, http.client.HTTPException):
|
||||
if deadline is None and attempt >= 3:
|
||||
raise
|
||||
if deadline is not None:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TaskEntryError("egress not open after 60 s") from None
|
||||
time.sleep(min(delay, remaining))
|
||||
delay = min(delay * 2, 8)
|
||||
else:
|
||||
time.sleep(attempt)
|
||||
|
||||
|
||||
def download(url, destination, *, readiness_budget=None):
|
||||
transfer(url, destination, readiness_budget=readiness_budget)
|
||||
|
||||
|
||||
def unpack(source, destination):
|
||||
"""Reject links and traversal; source archives contain only regular files/directories."""
|
||||
with tarfile.open(source) as archive:
|
||||
for member in archive.getmembers():
|
||||
target = (destination / member.name).resolve()
|
||||
if not target.is_relative_to(destination.resolve()) or not (member.isfile() or member.isdir()):
|
||||
raise ValueError("unsafe source archive member")
|
||||
archive.extractall(destination, filter="data")
|
||||
|
||||
|
||||
def failure_record(pin, reason):
|
||||
return {
|
||||
**pin,
|
||||
"done": True,
|
||||
"status": "failed",
|
||||
"reason": reason,
|
||||
"units": [
|
||||
{
|
||||
**unit,
|
||||
"status": "failed" if unit.get("supported", True) else "unsupported",
|
||||
"reason": reason,
|
||||
"instrument": {"files": [], "bytes": 0, "jvm": []},
|
||||
"questions": {},
|
||||
"questions_asked": [],
|
||||
"questions_completed": [],
|
||||
"witness_rows": [],
|
||||
"traces": [],
|
||||
}
|
||||
for unit in pin.get("units", [])
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main(*, work_root=Path("/tmp"), worker_script=None, readiness_budget=60):
|
||||
pin = {}
|
||||
failed = False
|
||||
output = work_root / "out"
|
||||
result = output / "result.json"
|
||||
step = "prepare output"
|
||||
try:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
result.unlink(missing_ok=True)
|
||||
questions = work_root / "questions.json"
|
||||
step = "GET questions"
|
||||
download(os.environ["QUESTIONS_URL"], questions, readiness_budget=readiness_budget)
|
||||
step = "parse questions"
|
||||
pin = json.loads(questions.read_text())
|
||||
if not isinstance(pin, dict):
|
||||
pin = {}
|
||||
raise ValueError("questions object must contain a prepared pin")
|
||||
source = work_root / "source.tar"
|
||||
step = "GET source"
|
||||
download(os.environ["SOURCE_URL"], source)
|
||||
step = "unpack source"
|
||||
case = work_root / "case"
|
||||
case.mkdir(parents=True, exist_ok=True)
|
||||
unpack(source, case)
|
||||
step = "configure worker"
|
||||
deadline = int(os.environ.get("DEADLINE", "900"))
|
||||
command = [
|
||||
sys.executable,
|
||||
str(worker_script or Path(__file__).with_name("engine_trace_worker.py")),
|
||||
"--worker",
|
||||
str(questions),
|
||||
"--root",
|
||||
str(case),
|
||||
"--output",
|
||||
str(result),
|
||||
"--heap-profile",
|
||||
"cluster",
|
||||
"--deadline",
|
||||
str(deadline),
|
||||
"--question-deadline",
|
||||
os.environ.get("QUESTION_DEADLINE", "120"),
|
||||
]
|
||||
step = "run worker"
|
||||
completed = subprocess.run(command, timeout=deadline + 5, check=False)
|
||||
if completed.returncode:
|
||||
raise TaskEntryError(f"worker exited {completed.returncode}")
|
||||
step = "read worker result"
|
||||
if not result.is_file() or not json.loads(result.read_text()).get("done"):
|
||||
raise TaskEntryError("worker did not write a completed result")
|
||||
except Exception as exc:
|
||||
failed = True
|
||||
# Avoid including exception text containing a presigned URL in durable output.
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
detail = f"HTTP {exc.code}"
|
||||
else:
|
||||
detail = str(exc) if isinstance(exc, TaskEntryError) else type(exc).__name__
|
||||
reason = f"{step}: {detail}"
|
||||
try:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
result.write_text(json.dumps(failure_record(pin, reason)) + "\n")
|
||||
except Exception:
|
||||
print("task entry could not write failure result", file=sys.stderr)
|
||||
step = "pack result"
|
||||
try:
|
||||
archive = work_root / "result.tar"
|
||||
with tarfile.open(archive, "w") as tar:
|
||||
for path in sorted(output.iterdir()):
|
||||
tar.add(path, arcname=path.name)
|
||||
data = archive.read_bytes()
|
||||
step = "PUT result"
|
||||
request = urllib.request.Request(os.environ["RESULT_URL"], data=data, method="PUT", headers={"Content-Type": "application/x-tar"})
|
||||
transfer(request, readiness_budget=readiness_budget if failed else None)
|
||||
except Exception as exc:
|
||||
detail = f"HTTP {exc.code}" if isinstance(exc, urllib.error.HTTPError) else type(exc).__name__
|
||||
print(f"{step}: {detail}; dispatcher deadline remains authoritative", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,108 +0,0 @@
|
||||
"""Engine workload adapter for the shared AX transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from engine_trace_k8s import pack
|
||||
|
||||
from benchmarks.ax.batch import AXLane, Item, SandboxFailure, Workload, manifest
|
||||
from benchmarks.ax.batch import task_state as task_state
|
||||
from openultrasast.plane.memory import open_store as open_store
|
||||
|
||||
|
||||
def task_manifest(name, args, urls):
|
||||
return manifest(
|
||||
name, engine_workload(args, lambda *_: None), engine_item({}, "pin", b"", args), urls, args.atespace, validate_name=False
|
||||
)
|
||||
|
||||
|
||||
def engine_workload(args, validate):
|
||||
return Workload(args.image, frozenset({"SOURCE_URL", "QUESTIONS_URL", "RESULT_URL"}), args.deadline, validate, kind="pass")
|
||||
|
||||
|
||||
def engine_item(pin, pin_id, source, args):
|
||||
return Item(
|
||||
pin_id,
|
||||
{"SOURCE_URL": source, "QUESTIONS_URL": json.dumps(pin).encode()},
|
||||
"result.dat",
|
||||
("python3", "/app/benchmarks/learn/engine_task_entry.py"),
|
||||
{"DEADLINE": str(args.deadline), "QUESTION_DEADLINE": str(args.question_deadline)},
|
||||
"",
|
||||
{"SOURCE_URL": "source.dat", "QUESTIONS_URL": "questions.json"},
|
||||
)
|
||||
|
||||
|
||||
class AX(AXLane):
|
||||
def __init__(self, args, store=None, pause=time.sleep, clock=time.monotonic):
|
||||
super().__init__(args, engine_workload(args, lambda *_: None), store, pause, clock)
|
||||
|
||||
def receive(self, data, output, pin):
|
||||
# Validate in isolation: malformed results must never become usable checkpoints.
|
||||
with tempfile.TemporaryDirectory(prefix="engine-ax-result-") as temporary:
|
||||
root = Path(temporary)
|
||||
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
||||
archive.extractall(root, filter="data")
|
||||
record = json.loads((root / "result.json").read_text())
|
||||
units = record.get("units", [])
|
||||
if record.get("done") and record.get("status") == "failed" and not units:
|
||||
# Questions download can fail before the actor learns any unit identities.
|
||||
# Keep that explicit failure, filling identities only from the host input.
|
||||
record = {
|
||||
**pin,
|
||||
**record,
|
||||
"units": [
|
||||
{
|
||||
**unit,
|
||||
"status": "failed" if unit["supported"] else "unsupported",
|
||||
"reason": record.get("reason", "AX task entry failed"),
|
||||
"instrument": {"files": [], "bytes": 0, "cpg_path": None, "cpg_bytes": 0, "jvm": []},
|
||||
"questions": {},
|
||||
"questions_asked": [],
|
||||
"questions_completed": [],
|
||||
"witness_rows": [],
|
||||
"traces": [],
|
||||
}
|
||||
for unit in pin["units"]
|
||||
],
|
||||
}
|
||||
units = record["units"]
|
||||
(root / "result.json").write_text(json.dumps(record))
|
||||
expected = [u["unit"] for u in pin["units"]]
|
||||
actual = [u["unit"] for u in units]
|
||||
if not record.get("done") or sorted(actual) != sorted(expected):
|
||||
raise ValueError("AX Task returned incomplete or mismatched units")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
# Keep the artifacts without interpreting any remote paths as host destinations.
|
||||
shutil.copytree(root, output, dirs_exist_ok=True)
|
||||
|
||||
def execute(self, checkout, output, pin, pin_id):
|
||||
import copy
|
||||
|
||||
self = copy.copy(self)
|
||||
self.workload = engine_workload(self.args, lambda data, target: self.receive(data, target, pin))
|
||||
attempts = []
|
||||
try:
|
||||
for attempt in range(2):
|
||||
try:
|
||||
return self._execute_attempt(engine_item(pin, pin_id, pack(checkout), self.args), output, attempt, attempts)
|
||||
except SandboxFailure:
|
||||
if attempt == 0:
|
||||
self.pause(10)
|
||||
return subprocess.CompletedProcess([], 1, "", "AX sandbox failed twice"), None
|
||||
finally:
|
||||
(output / "ax_attempts.json").write_text(json.dumps(attempts))
|
||||
result_path = output / "result.json"
|
||||
if result_path.exists():
|
||||
record = json.loads(result_path.read_text())
|
||||
record["ax_attempts"] = attempts
|
||||
result_path.write_text(json.dumps(record))
|
||||
@@ -1,259 +0,0 @@
|
||||
"""Host Kubernetes transport; the embedded bootstrap uses only the standard library."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from openultrasast.plane.memory import PRESIGNED_ARCHIVE_SUFFIX, S3Store, open_store
|
||||
|
||||
IMAGE = "ghcr.io/norandom/openultrasast:2.0.1"
|
||||
BOOTSTRAP = r"""
|
||||
import io, json, os, pathlib, subprocess, sys, tarfile, urllib.request
|
||||
|
||||
def get(name):
|
||||
with urllib.request.urlopen(os.environ[name], timeout=120) as response:
|
||||
return response.read()
|
||||
|
||||
def put(data):
|
||||
request = urllib.request.Request(os.environ['RESULT_URL'], data=data, method='PUT')
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
response.read()
|
||||
|
||||
try:
|
||||
if os.environ.get('CHECK') == '1':
|
||||
data = get('SOURCE_URL')
|
||||
assert len(data) == 1024
|
||||
subprocess.run([sys.executable, '-c', 'print(42)'], check=True)
|
||||
put(data)
|
||||
else:
|
||||
for name, target in [('SOURCE_URL', '/case'), ('ANALYZER_URL', '/frozen')]:
|
||||
pathlib.Path(target).mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(fileobj=io.BytesIO(get(name))) as archive:
|
||||
archive.extractall(target, filter='data')
|
||||
out = pathlib.Path('/out')
|
||||
out.mkdir(exist_ok=True)
|
||||
(out / 'pin.json').write_text(os.environ['PIN'])
|
||||
env = dict(os.environ, PYTHONPATH='/frozen/src')
|
||||
for key in ['SOURCE_URL', 'ANALYZER_URL', 'RESULT_URL']:
|
||||
env.pop(key, None)
|
||||
with (out / 'container.log').open('w') as log:
|
||||
done = subprocess.run([sys.executable, '/frozen/benchmarks/learn/engine_trace_worker.py',
|
||||
'--worker', '/out/pin.json', '--deadline', os.environ['DEADLINE'],
|
||||
'--question-deadline', os.environ['QUESTION_DEADLINE']], env=env, stdout=log, stderr=log)
|
||||
(out / 'exit.json').write_text(json.dumps(done.returncode))
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode='w') as archive:
|
||||
for path in out.iterdir():
|
||||
archive.add(path, arcname=path.name)
|
||||
put(buffer.getvalue())
|
||||
sys.exit(done.returncode)
|
||||
except Exception as exc:
|
||||
# urllib exceptions can contain bearer URLs. Never print exception text.
|
||||
print('engine bootstrap failed: ' + type(exc).__name__, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
|
||||
def pack(path):
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode="w") as archive:
|
||||
for child in sorted(path.iterdir()):
|
||||
archive.add(child, arcname=child.name)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def manifests(name, run, args, urls, pin, check=False):
|
||||
labels = {"engine-trace-run": run, "engine-trace-job": name, "engine-trace": "true"}
|
||||
secret = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "Secret",
|
||||
"metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
|
||||
"type": "Opaque",
|
||||
"stringData": urls,
|
||||
}
|
||||
env = [{"name": key, "valueFrom": {"secretKeyRef": {"name": name, "key": key}}} for key in urls]
|
||||
env += [
|
||||
{"name": key, "value": value}
|
||||
for key, value in {
|
||||
"PIN": json.dumps(pin),
|
||||
"DEADLINE": str(args.deadline),
|
||||
"QUESTION_DEADLINE": str(args.question_deadline),
|
||||
"CHECK": "1" if check else "0",
|
||||
}.items()
|
||||
]
|
||||
job = {
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
|
||||
"spec": {
|
||||
"activeDeadlineSeconds": args.deadline,
|
||||
"backoffLimit": 0,
|
||||
"ttlSecondsAfterFinished": 86400,
|
||||
"template": {
|
||||
"metadata": {"labels": labels},
|
||||
"spec": {
|
||||
"restartPolicy": "Never",
|
||||
"automountServiceAccountToken": False,
|
||||
"securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000},
|
||||
"volumes": [{"name": name, "emptyDir": {}} for name in ("case", "frozen", "out")],
|
||||
"affinity": {
|
||||
"podAntiAffinity": {
|
||||
"requiredDuringSchedulingIgnoredDuringExecution": [
|
||||
{
|
||||
"labelSelector": {"matchLabels": {"engine-trace": "true"}},
|
||||
"topologyKey": "kubernetes.io/hostname",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"containers": [
|
||||
{
|
||||
"name": "engine",
|
||||
"image": args.image,
|
||||
"command": ["python", "-c", BOOTSTRAP],
|
||||
"env": env,
|
||||
"volumeMounts": [{"name": name, "mountPath": "/" + name} for name in ("case", "frozen", "out")],
|
||||
"resources": {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}},
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return job, secret
|
||||
|
||||
|
||||
def redact(text):
|
||||
return re.sub(r"https?://[^\s\"'<>]+", "[redacted URL]", text)
|
||||
|
||||
|
||||
class Kubernetes:
|
||||
def __init__(self, args, store=None):
|
||||
self.args = args
|
||||
self.run = uuid.uuid4().hex
|
||||
self.store = store if store is not None else open_store()
|
||||
if not isinstance(self.store, S3Store):
|
||||
raise ValueError("Kubernetes requires OUSAST_MEMORY=s3://bucket[/prefix]")
|
||||
|
||||
def kubectl(self, *args, manifest=None, check=True):
|
||||
result = subprocess.run(
|
||||
["kubectl", "--context", self.args.kube_context, "--namespace", self.args.kube_namespace, *args],
|
||||
input=json.dumps(manifest) if manifest is not None else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if check and result.returncode:
|
||||
# kubectl can echo submitted Secret data on errors; never expose stderr.
|
||||
raise ValueError(f"kubectl {args[0]} failed (exit {result.returncode})")
|
||||
return result
|
||||
|
||||
def namespace(self):
|
||||
self.kubectl("get", "nodes", "-o", "json")
|
||||
self.kubectl("apply", "-f", "-", manifest={"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": self.args.kube_namespace}})
|
||||
|
||||
def execute(self, source, checkout, output, pin, check=False):
|
||||
name = "engine-" + uuid.uuid4().hex[:24]
|
||||
pin_key = hashlib.sha256(json.dumps([pin.get("repo"), pin.get("pin")]).encode()).hexdigest()
|
||||
prefix = f"engine-jobs/{self.run}/{pin_key}/"
|
||||
keys = [prefix + name + PRESIGNED_ARCHIVE_SUFFIX for name in ("source", "analyzer", "result")]
|
||||
expiry = timedelta(seconds=self.args.deadline + 900)
|
||||
node = None
|
||||
try:
|
||||
self.store._put(keys[0], b"x" * 1024 if check else pack(checkout))
|
||||
self.store._put(keys[1], b"" if check else pack(source))
|
||||
urls = {
|
||||
"SOURCE_URL": self.store.presign_get(keys[0], expiry),
|
||||
"ANALYZER_URL": self.store.presign_get(keys[1], expiry),
|
||||
"RESULT_URL": self.store.presign_put(keys[2], expiry),
|
||||
}
|
||||
job, secret = manifests(name, self.run, self.args, urls, pin, check)
|
||||
# create avoids kubectl's last-applied annotation duplicating Secret values.
|
||||
created = json.loads(self.kubectl("create", "-f", "-", "-o", "json", manifest=job).stdout)
|
||||
secret["metadata"]["ownerReferences"] = [
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"name": name,
|
||||
"uid": created["metadata"]["uid"],
|
||||
}
|
||||
]
|
||||
self.kubectl("create", "-f", "-", manifest=secret)
|
||||
end = time.monotonic() + self.args.deadline + 60
|
||||
reason = "host deadline waiting for Job"
|
||||
while time.monotonic() < end:
|
||||
state = json.loads(self.kubectl("get", "job", name, "-o", "json").stdout).get("status", {})
|
||||
pods = json.loads(self.kubectl("get", "pods", "-l", f"engine-trace-job={name}", "-o", "json").stdout)
|
||||
node = next((p.get("spec", {}).get("nodeName") for p in pods.get("items", []) if p.get("spec", {}).get("nodeName")), node)
|
||||
result = self.store._get(keys[2])
|
||||
failed = next((c for c in state.get("conditions", []) if c.get("type") == "Failed" and c.get("status") == "True"), None)
|
||||
if failed or state.get("failed"):
|
||||
reason = (failed or {}).get("reason", "Job failed") + ": " + (failed or {}).get("message", "")
|
||||
break
|
||||
if state.get("succeeded") and result is not None:
|
||||
if check:
|
||||
if result[0] != b"x" * 1024:
|
||||
raise ValueError("round-trip bytes mismatch")
|
||||
else:
|
||||
with tarfile.open(fileobj=io.BytesIO(result[0])) as archive:
|
||||
archive.extractall(output, filter="data")
|
||||
if not (output / "result.json").is_file():
|
||||
raise ValueError("Job produced no result.json")
|
||||
record = json.loads((output / "result.json").read_text())
|
||||
if not record.get("done") or {u["unit"] for u in record["units"]} != {u["unit"] for u in pin["units"]}:
|
||||
raise ValueError("Job returned incomplete or mismatched units")
|
||||
return subprocess.CompletedProcess([], 0, "", ""), node
|
||||
time.sleep(2)
|
||||
logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
|
||||
reason = redact(reason + "\n" + logs.stdout + logs.stderr)
|
||||
return subprocess.CompletedProcess([], 1, "", reason), node
|
||||
except Exception as exc:
|
||||
reason = "Kubernetes transport failed: " + type(exc).__name__ + ": " + redact(str(exc))
|
||||
try:
|
||||
logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
|
||||
reason += "\n" + redact(logs.stdout + logs.stderr)
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
return subprocess.CompletedProcess([], 1, "", reason), node
|
||||
finally:
|
||||
if not self.args.keep_jobs:
|
||||
# Attempt every cleanup even if an earlier deletion fails.
|
||||
errors = []
|
||||
for kind in ("job", "secret"):
|
||||
try:
|
||||
self.kubectl("delete", kind, name, "--ignore-not-found", "--wait=true", "--timeout=30s")
|
||||
except Exception as exc:
|
||||
errors.append(type(exc).__name__)
|
||||
for key in keys:
|
||||
try:
|
||||
self.store._delete(key)
|
||||
except Exception as exc:
|
||||
errors.append(type(exc).__name__)
|
||||
if errors:
|
||||
raise RuntimeError("engine Job cleanup failed: " + ", ".join(errors))
|
||||
|
||||
def check(self):
|
||||
self.namespace()
|
||||
nodes = json.loads(self.kubectl("get", "nodes", "-o", "json").stdout)
|
||||
for node in nodes["items"]:
|
||||
print(json.dumps({"node": node["metadata"]["name"], "allocatable_memory": node["status"]["allocatable"]["memory"]}))
|
||||
done, node = self.execute(None, None, None, {}, check=True)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"k8s_check": "passed" if done.returncode == 0 else "failed",
|
||||
"node": node,
|
||||
"reason": done.stderr,
|
||||
"round_trip_bytes": 1024 if done.returncode == 0 else 0,
|
||||
}
|
||||
)
|
||||
)
|
||||
return done.returncode
|
||||
@@ -1,51 +0,0 @@
|
||||
"""Lossless parsing of the optional shortest-path trace, with explicit evidence limits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
_STEP = re.compile(r"^(.*):(-?\d+) \[([^\]]+)\] ([\s\S]*)$")
|
||||
|
||||
|
||||
def parse_trace(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Preserve row-level discharge evidence; never guess which node sanitized a path.
|
||||
|
||||
The query emits basename-only files and truncated code. A bound can be off-path;
|
||||
it remains in ``guard`` even when no trace step exactly matches it. Invalid steps
|
||||
raise instead of turning an unreadable trace into an empty path.
|
||||
"""
|
||||
raw = row.get("trace", [])
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("trace must be an array")
|
||||
steps = []
|
||||
bound = str(row.get("bound") or "")
|
||||
for index, text in enumerate(raw):
|
||||
match = _STEP.fullmatch(text) if isinstance(text, str) else None
|
||||
if match is None:
|
||||
raise ValueError(f"invalid trace step: {text!r}")
|
||||
location, line, node_label, code = match.groups()
|
||||
file, separator, method = location.partition(":")
|
||||
roles = ["source"] if index == 0 else ["step"]
|
||||
if index == len(raw) - 1:
|
||||
roles = ["source", "sink"] if index == 0 else ["sink"]
|
||||
if bound and code == bound:
|
||||
roles.append("guard")
|
||||
steps.append(
|
||||
{
|
||||
"file": file,
|
||||
"method": method if separator else "?",
|
||||
"line": int(line),
|
||||
"node_label": node_label,
|
||||
"code": code,
|
||||
"roles": roles,
|
||||
"raw": text,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"steps": steps,
|
||||
"sanitized": bool(row.get("sanitized", False)),
|
||||
"sanitizer": {"on_path": bool(row.get("sanitized", False)), "step": None},
|
||||
"guard": {"text": bound, "bounded": bool(row.get("bounded", False))},
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Host queue transport: source and task metadata only, never analyzer code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from engine_worker_service import PENDING, client_from_env, pack, put, read, result_key, status
|
||||
|
||||
from openultrasast.plane.memory import PRESIGNED_ARCHIVE_SUFFIX
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
def __init__(self, args, client=None, *, pause=time.sleep, clock=time.monotonic):
|
||||
self.client = client if client is not None else client_from_env()
|
||||
self.run = uuid.uuid4().hex
|
||||
self.args, self.pause, self.clock = args, pause, clock
|
||||
|
||||
def execute(self, checkout, output, pin, pin_id):
|
||||
task_id = f"{self.run}-{pin_id}"
|
||||
task = dict(
|
||||
run=self.run,
|
||||
id=pin_id,
|
||||
task_id=task_id,
|
||||
pin=pin,
|
||||
input=f"engine-queue/inputs/{self.run}/{pin_id}/source{PRESIGNED_ARCHIVE_SUFFIX}",
|
||||
image=self.args.image,
|
||||
deadline=self.args.deadline,
|
||||
question_deadline=self.args.question_deadline,
|
||||
)
|
||||
try:
|
||||
self.client.put(task["input"], pack(checkout), {})
|
||||
put(self.client, PENDING + task_id, task)
|
||||
end = self.clock() + self.args.queue_timeout
|
||||
while self.clock() < end:
|
||||
result = read(self.client, result_key(task))
|
||||
if result is not None:
|
||||
(output / "result.json").write_text(json.dumps(result))
|
||||
self.client.delete(task["input"])
|
||||
return subprocess.CompletedProcess([], 0, "", ""), result.get("worker")
|
||||
self.pause(2)
|
||||
# Removing pending cancels work that has not been claimed; retain source for a live lease.
|
||||
self.client.delete(PENDING + task_id)
|
||||
return None, None
|
||||
except Exception as exc:
|
||||
# SDK errors can contain endpoint URLs or credentials. Do not persist their text.
|
||||
raise OSError("queue transport failed") from exc
|
||||
|
||||
def status(self):
|
||||
return status(self.client)
|
||||
@@ -1,644 +0,0 @@
|
||||
"""Container-only Joern trace worker; host prepares all corpus and question metadata.
|
||||
|
||||
Keep this module independent of learn/plane and their optional dependencies. The
|
||||
frozen host invokes this file directly in the shipped, offline analyzer image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from engine_trace_parse import parse_trace
|
||||
|
||||
from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, _render, extract_payload
|
||||
from openultrasast.cpg.session import EngineSession
|
||||
from openultrasast.model.contracts import ExecutionBudget
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def safe_path(root, name):
|
||||
path = (root / name).resolve()
|
||||
if not path.is_relative_to(root.resolve()):
|
||||
raise ValueError(f"path escapes input root: {name}")
|
||||
return path
|
||||
|
||||
|
||||
FRONTENDS = {"php2cpg", "pysrc2cpg", "jssrc2cpg", "javasrc2cpg", "c2cpg", "joern-parse"}
|
||||
MIN_CPG_BYTES = 1024
|
||||
HEAP_MB = 2560
|
||||
CLUSTER_JVM_FLAGS = (
|
||||
"-Xms1700m -Xmx1700m -XX:MaxMetaspaceSize=256m -XX:ReservedCodeCacheSize=128m "
|
||||
"-XX:MaxDirectMemorySize=256m -Xss512k -XX:ActiveProcessorCount=2 -XX:+UseG1GC"
|
||||
)
|
||||
ENGINE_BUG = "engine bug: jssrc2cpg ObjectPropertyCallLinker"
|
||||
EXCLUDED_DIRS = frozenset({"tests", "test", "docs", "node_modules", "vendor", "dist", "build", "templates", "migrations"})
|
||||
|
||||
|
||||
def instrument_failure(proof):
|
||||
if any(stage.get("oom") for stage in proof.get("jvm", [])):
|
||||
return "JVM out of memory"
|
||||
if not proof.get("files") or not proof.get("bytes"):
|
||||
return "instrument read zero files or zero bytes"
|
||||
if proof.get("cpg_bytes", 0) <= MIN_CPG_BYTES:
|
||||
return "frontend wrote no CPG"
|
||||
for stage in proof.get("jvm", []):
|
||||
if stage["command"] in FRONTENDS:
|
||||
if stage.get("cpg_bytes", 0) <= MIN_CPG_BYTES:
|
||||
return "frontend wrote no CPG"
|
||||
elif (
|
||||
stage["command"] == "joern" or (stage["command"] == "joern-session" and stage.get("step") == "start" and stage.get("success"))
|
||||
) and stage["seconds"] < 5:
|
||||
return f"implausibly fast JVM step: {stage['command']} in {stage['seconds']:.3f}s (<5s)"
|
||||
if not proof.get("jvm"):
|
||||
return "no JVM step recorded"
|
||||
return ""
|
||||
|
||||
|
||||
class MeasuredBackend(JoernBackend):
|
||||
"""Use the shipped backend with process and resident-session watchdogs."""
|
||||
|
||||
def __init__(self, deadline, question_deadline, checkpoint, heap_profile="vm"):
|
||||
if heap_profile not in {"vm", "cluster"}:
|
||||
raise ValueError(f"unknown heap profile: {heap_profile}")
|
||||
self.heap_profile = heap_profile
|
||||
super().__init__(
|
||||
build_timeout=deadline,
|
||||
query_timeout=deadline,
|
||||
session_transport=False,
|
||||
heap_mb=1700 if heap_profile == "cluster" else HEAP_MB,
|
||||
)
|
||||
self.deadline = time.monotonic() + deadline
|
||||
self.question_deadline = question_deadline
|
||||
self.checkpoint = checkpoint
|
||||
self.stages = []
|
||||
self.asked = []
|
||||
self.answers = {}
|
||||
self.completion_counts = Counter()
|
||||
self.timed_out = False
|
||||
self.fatal = ""
|
||||
self.labelled_files = []
|
||||
self.retry_pending = False
|
||||
self.retries = []
|
||||
self.session_fallback = ""
|
||||
|
||||
def _jvm_env(self):
|
||||
env = super()._jvm_env()
|
||||
if self.heap_profile == "cluster":
|
||||
# Bound every JVM, including the child spawned by the Joern launcher.
|
||||
env["JAVA_TOOL_OPTIONS"] = CLUSTER_JVM_FLAGS
|
||||
env.pop("_JAVA_OPTIONS", None)
|
||||
env.pop("JDK_JAVA_OPTIONS", None)
|
||||
return env
|
||||
|
||||
def _session_stage(self, name, action):
|
||||
started = time.monotonic()
|
||||
result = None
|
||||
try:
|
||||
result = action()
|
||||
return result
|
||||
finally:
|
||||
self.stages.append(
|
||||
{
|
||||
"command": "joern-session",
|
||||
"step": name,
|
||||
"success": result is not None and result is not False,
|
||||
"seconds": time.monotonic() - started,
|
||||
"heap_mb": self._heap_mb(),
|
||||
"java_tool_options": self._jvm_env()["JAVA_TOOL_OPTIONS"],
|
||||
}
|
||||
)
|
||||
self.checkpoint()
|
||||
|
||||
def _fallback_session(self, reason):
|
||||
self.session_fallback = reason
|
||||
if self._session is not None:
|
||||
detail = self._session.log_tail()
|
||||
if "OutOfMemoryError" in detail:
|
||||
self.fatal = "JVM out of memory"
|
||||
self.stages[-1]["oom"] = True
|
||||
self.stages[-1]["output_tail"] = detail
|
||||
self.close_session()
|
||||
self.checkpoint()
|
||||
|
||||
def _session_for(self, cpg_path):
|
||||
if self.session_fallback or self.fatal or self.timed_out:
|
||||
return None
|
||||
try:
|
||||
if self._session is None:
|
||||
self._session = EngineSession(
|
||||
cpg_path.parent / "engine-session",
|
||||
ExecutionBudget(self.deadline, 5),
|
||||
self._jvm_env(),
|
||||
startup_allowance_seconds=min(180, max(20, (self.deadline - time.monotonic()) / 3)),
|
||||
)
|
||||
if not self._session_stage("start", self._session.start):
|
||||
self._fallback_session(self._session.failure or "session_start_failed")
|
||||
return None
|
||||
if self._session.loaded_graph != str(cpg_path):
|
||||
if not self._session_stage("load", lambda: self._session.load(cpg_path, timeout=self.query_timeout)):
|
||||
self._fallback_session(self._session.failure or "session_load_failed")
|
||||
return None
|
||||
self._defined.clear()
|
||||
return self._session
|
||||
except (OSError, RuntimeError) as exc:
|
||||
self._fallback_session(f"session_start_or_load:{exc}")
|
||||
return None
|
||||
|
||||
def _session_payload(self, cpg_path, query, params):
|
||||
session = self._session_for(cpg_path)
|
||||
if session is None:
|
||||
return None
|
||||
end = min(self.deadline, time.monotonic() + self.query_timeout)
|
||||
question_end = end
|
||||
prior_asked = list(self.asked)
|
||||
prior_answers = dict(self.answers)
|
||||
prior_counts = self.completion_counts.copy()
|
||||
consumed = 0
|
||||
streamed = ""
|
||||
|
||||
def observe(output):
|
||||
nonlocal consumed, question_end, streamed
|
||||
streamed = output
|
||||
lines = output[consumed:].splitlines(keepends=True)
|
||||
for line in lines:
|
||||
if not line.endswith("\n"):
|
||||
break
|
||||
consumed += len(line)
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
if "__question__" in event:
|
||||
self.asked.append(event["__question__"])
|
||||
question_end = min(end, time.monotonic() + self.question_deadline)
|
||||
if "id" in event and "rows" in event:
|
||||
self.answers[event["id"]] = event["rows"]
|
||||
self.completion_counts[event["id"]] += 1
|
||||
question_end = end
|
||||
self.checkpoint()
|
||||
if time.monotonic() >= min(end, question_end):
|
||||
self.timed_out = True
|
||||
return False
|
||||
return True
|
||||
|
||||
def run():
|
||||
source = (self.queries_dir / f"{query}.sc").read_text()
|
||||
# Same shipped script/parameters as JoernBackend, using the graph loaded above.
|
||||
if source.count("importCpg(cpgFile)") != 1:
|
||||
return None
|
||||
symbol = f"ousast_{query}"
|
||||
if query not in self._defined:
|
||||
source = source.replace("importCpg(cpgFile)", "()", 1)
|
||||
source = source.replace("@main def exec(", f"def {symbol}(", 1)
|
||||
if session.define(source, timeout=max(0, end - time.monotonic())) is None:
|
||||
return None
|
||||
self._defined.add(query)
|
||||
arguments = ", ".join(f"{name} = {json.dumps(_render(value))}" for name, value in sorted(params.items()))
|
||||
session.output_observer = observe
|
||||
try:
|
||||
answer = session.evaluate(f"{symbol}({arguments})", timeout=max(0, end - time.monotonic()))
|
||||
finally:
|
||||
session.output_observer = None
|
||||
if answer is not None:
|
||||
observe(answer.stdout)
|
||||
return answer.stdout
|
||||
return None
|
||||
|
||||
failure = ""
|
||||
try:
|
||||
body = self._session_stage(query, run)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
body = None
|
||||
failure = f"session_{query}:{exc}"
|
||||
if self.timed_out:
|
||||
self.close_session()
|
||||
return streamed
|
||||
parsed = extract_payload(body) if body is not None else None
|
||||
if not isinstance(parsed, dict):
|
||||
# Discard markers from an unverified attempt before replaying the script.
|
||||
self.asked[:] = prior_asked
|
||||
self.answers = prior_answers
|
||||
self.completion_counts = prior_counts
|
||||
self._fallback_session(failure or session.failure or f"session_{query}_payload_missing")
|
||||
return None
|
||||
if query == "census":
|
||||
self.stages[-1]["files"] = int(parsed.get("files", 0))
|
||||
if self.stages[-1]["files"] <= 0:
|
||||
self.fatal = "query census read zero files"
|
||||
return body
|
||||
|
||||
def _apply_overlays(self, cpg_path, scratch):
|
||||
session = self._session_for(cpg_path)
|
||||
result = False
|
||||
if session is not None:
|
||||
body = self._session_payload(cpg_path, "overlay", {"cpgFile": str(cpg_path)})
|
||||
saved = session.scratch / "workspace" / cpg_path.name / "cpg.bin"
|
||||
if body is not None and saved.is_file():
|
||||
import shutil
|
||||
|
||||
shutil.copyfile(saved, cpg_path)
|
||||
result = True
|
||||
elif body is not None:
|
||||
self._fallback_session("session_overlay_saved_graph_missing")
|
||||
if not result:
|
||||
result = super()._apply_overlays(cpg_path, scratch)
|
||||
# An optional overlay failure must not prevent the taint-only attempt.
|
||||
if self.fatal == ENGINE_BUG and not self.timed_out:
|
||||
self.retry_pending = True
|
||||
self.fatal = ""
|
||||
return result
|
||||
|
||||
def _batch_once(self, cpg_path, query, requests):
|
||||
if self.fatal or self.timed_out:
|
||||
return None
|
||||
if not self.retry_pending:
|
||||
answers = super()._batch_once(cpg_path, query, requests)
|
||||
if self.fatal != ENGINE_BUG:
|
||||
return answers
|
||||
if query != "taint" or self.timed_out or self.retries:
|
||||
self.fatal = ENGINE_BUG
|
||||
return None
|
||||
self.retry_pending = False
|
||||
self.fatal = ""
|
||||
retry = {"reason": ENGINE_BUG, "mode": "taint-only", "completed": False}
|
||||
self.retries.append(retry)
|
||||
# Do not reuse rows/markers from the failed attempt as completion evidence.
|
||||
self.asked.clear()
|
||||
self.answers.clear()
|
||||
self.completion_counts.clear()
|
||||
original = self.queries_dir
|
||||
with tempfile.TemporaryDirectory(prefix="trace-taint-", dir=cpg_path.parent) as scratch:
|
||||
script = (original / "taint.sc").read_text()
|
||||
marker = "importCpg(cpgFile)"
|
||||
if script.count(marker) != 1:
|
||||
self.fatal = ENGINE_BUG
|
||||
return None
|
||||
# Loading without enhancement skips the failing JS postprocessing passes.
|
||||
# A raw graph without dataflow cannot establish a negative answer.
|
||||
script = script.replace(
|
||||
marker,
|
||||
"importCpg(cpgFile, enhance = false)\n"
|
||||
f' require(cpg.metaData.overlays.l.contains("{DATAFLOW_OVERLAY}"), '
|
||||
'"taint-only retry requires saved dataflowOss overlay")',
|
||||
)
|
||||
self.queries_dir = Path(scratch)
|
||||
(self.queries_dir / "taint.sc").write_text(script)
|
||||
try:
|
||||
answers = super()._batch_once(cpg_path, query, requests)
|
||||
finally:
|
||||
self.queries_dir = original
|
||||
retry["completed"] = answers is not None and not self.fatal and not self.timed_out
|
||||
if not retry["completed"]:
|
||||
self.fatal = ENGINE_BUG
|
||||
return answers
|
||||
|
||||
def _run(self, command, *, timeout, cwd=None):
|
||||
if self.fatal or self.timed_out:
|
||||
return None
|
||||
if time.monotonic() >= self.deadline:
|
||||
self.timed_out = True
|
||||
return None
|
||||
started = time.monotonic()
|
||||
end = min(self.deadline, started + timeout)
|
||||
question_end = end
|
||||
binary = Path(command[0]).name
|
||||
frontend = binary in FRONTENDS
|
||||
query = binary == "joern" and "--script" in command
|
||||
output = bytearray()
|
||||
pending = b""
|
||||
process = subprocess.Popen(
|
||||
command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=self._jvm_env(), start_new_session=True
|
||||
)
|
||||
try:
|
||||
with selectors.DefaultSelector() as selector:
|
||||
selector.register(process.stdout, selectors.EVENT_READ)
|
||||
while selector.get_map():
|
||||
if time.monotonic() >= min(end, question_end):
|
||||
self.timed_out = True
|
||||
break
|
||||
for key, _ in selector.select(min(0.2, max(0, min(end, question_end) - time.monotonic()))):
|
||||
chunk = os.read(key.fileobj.fileno(), 65536)
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
output.extend(chunk)
|
||||
pending += chunk
|
||||
while b"\n" in pending:
|
||||
line, pending = pending.split(b"\n", 1)
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
if "__question__" in event:
|
||||
self.asked.append(event["__question__"])
|
||||
question_end = min(end, time.monotonic() + self.question_deadline)
|
||||
if "id" in event and "rows" in event:
|
||||
self.answers[event["id"]] = event["rows"]
|
||||
self.completion_counts[event["id"]] += 1
|
||||
question_end = end
|
||||
self.checkpoint()
|
||||
if not self.timed_out:
|
||||
process.wait(timeout=max(0.01, end - time.monotonic()))
|
||||
finally:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
elapsed = time.monotonic() - started
|
||||
if frontend or query:
|
||||
stage = {
|
||||
"command": binary,
|
||||
"argv": command,
|
||||
"heap_mb": self._heap_mb(),
|
||||
"java_tool_options": self._jvm_env()["JAVA_TOOL_OPTIONS"],
|
||||
"seconds": elapsed,
|
||||
"exit": process.returncode,
|
||||
"oom": b"OutOfMemoryError" in output,
|
||||
}
|
||||
lines = output.decode(errors="replace").splitlines()
|
||||
stage["labelled_file_logs"] = {
|
||||
file: [line for line in lines if file in line or Path(file).name in line]
|
||||
for file in self.labelled_files
|
||||
if any(file in line or Path(file).name in line for line in lines)
|
||||
}
|
||||
self.stages.append(stage)
|
||||
if query and process.returncode != 0:
|
||||
# stderr is merged into stdout above; drop JVM frames before taking
|
||||
# the tail so they cannot crowd out the exception's message.
|
||||
lines = [
|
||||
line.strip()
|
||||
for line in output.decode(errors="replace").splitlines()
|
||||
if line.strip() and not line.strip().startswith(("at ", "... "))
|
||||
]
|
||||
stage["output_tail"] = "\n".join(lines[-20:])
|
||||
if not self.timed_out:
|
||||
self.fatal = f"joern query exited {process.returncode}"
|
||||
if frontend:
|
||||
# Capture the exact frontend output before build failure cleanup or overlays.
|
||||
output_arg = next((i + 1 for i, arg in enumerate(command[:-1]) if arg in {"-o", "--output"}), None)
|
||||
cpg_path = (Path(cwd or Path.cwd()) / command[output_arg]).resolve() if output_arg is not None else None
|
||||
stage["cpg_path"] = str(cpg_path) if cpg_path is not None else None
|
||||
stage["cpg_bytes"] = cpg_path.stat().st_size if cpg_path is not None and cpg_path.is_file() else 0
|
||||
if stage["cpg_bytes"] <= MIN_CPG_BYTES:
|
||||
self.fatal = "frontend wrote no CPG"
|
||||
elif elapsed < 5 and not self.fatal:
|
||||
self.fatal = f"implausibly fast JVM step: {command[0]} in {elapsed:.3f}s (<5s)"
|
||||
if b"ObjectPropertyCallLinker failed" in output and b"Assignment statement with 3 arguments" in output:
|
||||
self.fatal = ENGINE_BUG
|
||||
stage["engine_bug"] = ENGINE_BUG
|
||||
if b"OutOfMemoryError" in output:
|
||||
self.fatal = "JVM out of memory"
|
||||
self.checkpoint()
|
||||
return subprocess.CompletedProcess(command, process.returncode if not self.fatal else 1, output.decode(errors="replace"), "")
|
||||
|
||||
|
||||
def unit_record(unit, status, reason="", **extra):
|
||||
return {
|
||||
**unit,
|
||||
"status": status,
|
||||
"reason": reason,
|
||||
"witness_rows": [],
|
||||
"traces": [],
|
||||
"instrument": {"files": [], "bytes": 0, "cpg_path": None, "cpg_bytes": 0, "jvm": []},
|
||||
"questions": {},
|
||||
"questions_asked": [],
|
||||
"questions_completed": [],
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def worker_language(pin, root, out, deadline, question_deadline, persist=write_json, heap_profile="vm"):
|
||||
proof = {
|
||||
"files": [],
|
||||
"bytes": 0,
|
||||
"cpg_path": None,
|
||||
"cpg_bytes": 0,
|
||||
"jvm": [],
|
||||
"heap_mb": 1700 if heap_profile == "cluster" else HEAP_MB,
|
||||
"heap_profile": heap_profile,
|
||||
"java_tool_options": CLUSTER_JVM_FLAGS
|
||||
if heap_profile == "cluster"
|
||||
else f"{os.environ.get('JAVA_TOOL_OPTIONS', '').strip()} -Xmx{HEAP_MB}m".strip(),
|
||||
"excludes": sorted(EXCLUDED_DIRS),
|
||||
"materialization": pin.get("materialization", {}),
|
||||
}
|
||||
record = {
|
||||
**pin,
|
||||
"done": False,
|
||||
"instrument": proof,
|
||||
"units": [unit_record(u, "unsupported" if not u["supported"] else "timeout", u["reason"]) for u in pin["units"]],
|
||||
}
|
||||
|
||||
def checkpoint():
|
||||
# Checkpoints retain raw completed rows even if the outer container deadline fires.
|
||||
# They remain timeout until the finished JVM's instrument proof is validated.
|
||||
for stage in backend.stages:
|
||||
if "cpg_path" in stage:
|
||||
proof["cpg_path"] = stage["cpg_path"]
|
||||
proof["cpg_bytes"] = stage["cpg_bytes"]
|
||||
if getattr(backend, "session_fallback", ""):
|
||||
proof["session_fallback"] = backend.session_fallback
|
||||
for row in record["units"]:
|
||||
rid = row["unit"]
|
||||
row["instrument"] = proof
|
||||
row["questions_asked"] = [rid] if rid in backend.asked else []
|
||||
if rid in backend.answers:
|
||||
row["questions_completed"] = [rid]
|
||||
row["witness_rows"] = [r for r in backend.answers[rid] if isinstance(r, dict) and "kind" not in r and "sink" in r]
|
||||
persist(out, record)
|
||||
|
||||
backend = MeasuredBackend(deadline, question_deadline, checkpoint, **({"heap_profile": heap_profile} if heap_profile != "vm" else {}))
|
||||
proof["jvm"] = backend.stages
|
||||
proof["retries"] = getattr(backend, "retries", [])
|
||||
backend.labelled_files = sorted({u["file"] for u in pin["units"] if u["supported"]})
|
||||
cpg = None
|
||||
try:
|
||||
active = [u for u in pin["units"] if u["supported"]]
|
||||
for file in sorted({u["file"] for u in active}):
|
||||
data = safe_path(root, file).read_bytes()
|
||||
proof["files"].append({"file": file, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
|
||||
proof["bytes"] += len(data)
|
||||
print(f"read {file}: {len(data)} bytes", flush=True)
|
||||
if not proof["bytes"]:
|
||||
raise ValueError("instrument read zero files or zero bytes")
|
||||
languages = {"javascript" if u["language"] == "typescript" else u["language"] for u in active}
|
||||
if len(languages) != 1:
|
||||
raise ValueError(f"one pin has multiple frontend languages: {sorted(languages)}")
|
||||
language = next(iter(languages))
|
||||
requests = {rid: {key: _render(value) for key, value in params.items()} for rid, params in pin["questions"].items()}
|
||||
if set(requests) != {u["unit"] for u in active}:
|
||||
raise ValueError("prepared questions do not match supported units")
|
||||
record["questions"] = requests
|
||||
for unit_row in record["units"]:
|
||||
unit_row["questions"] = {unit_row["unit"]: requests[unit_row["unit"]]} if unit_row["unit"] in requests else {}
|
||||
persist(out, record)
|
||||
cpg = backend.build(root, language=language)
|
||||
if cpg is None:
|
||||
raise ValueError(backend.fatal or backend.last_failure or ("pin deadline" if backend.timed_out else "CPG build failed"))
|
||||
if proof["cpg_path"] is None:
|
||||
proof["cpg_path"] = str(cpg.cpg_path)
|
||||
proof["cpg_bytes"] = cpg.cpg_path.stat().st_size if cpg.cpg_path.is_file() else 0
|
||||
failure = instrument_failure(proof)
|
||||
if failure:
|
||||
raise ValueError(failure)
|
||||
if cpg.run_batch_once is None:
|
||||
raise ValueError("backend offers no batch operation")
|
||||
answers = cpg.run_batch_once("taint", requests) or {}
|
||||
record["census"] = answers.get("__census__", [])
|
||||
failure = backend.fatal or instrument_failure(proof)
|
||||
if not backend.asked and not backend.timed_out:
|
||||
failure = failure or "query produced no markers"
|
||||
census = record["census"]
|
||||
if census and int(census[0].get("files", 0)) <= 0:
|
||||
failure = failure or "query census read zero files"
|
||||
elif not census and not backend.timed_out:
|
||||
failure = failure or "query census is missing"
|
||||
for index, unit in enumerate(pin["units"]):
|
||||
if not unit["supported"]:
|
||||
continue
|
||||
rid = unit["unit"]
|
||||
extra = {
|
||||
"questions": {rid: requests[rid]},
|
||||
"questions_asked": [rid] if rid in backend.asked else [],
|
||||
"questions_completed": [rid] if rid in answers else [],
|
||||
"instrument": proof,
|
||||
}
|
||||
rows = answers.get(rid)
|
||||
# A batch across backend shards is complete only when every shard answered.
|
||||
# The legacy merger can return a union even when one shard was killed.
|
||||
shards = int(census[0].get("shards", 1)) if census else 1
|
||||
shard_incomplete = shards > 1 and getattr(backend, "completion_counts", {}).get(rid, 0) < shards
|
||||
if rows is not None:
|
||||
raw_flows = [r for r in rows if isinstance(r, dict) and "kind" not in r and "sink" in r]
|
||||
extra.update(witness_rows=raw_flows, traces=[parse_trace(r) for r in raw_flows])
|
||||
if shard_incomplete:
|
||||
extra["questions_completed"] = []
|
||||
names = census[0].get("file_names", []) if census else []
|
||||
file_missing = not any(str(p).removeprefix("./") == unit["file"] or str(p).endswith("/" + unit["file"]) for p in names)
|
||||
dropped = file_missing and rows is not None
|
||||
if failure or dropped:
|
||||
status, reason = "failed", failure or "labelled file was dropped or absent from query census"
|
||||
if dropped:
|
||||
logs = [line for stage in backend.stages for line in stage.get("labelled_file_logs", {}).get(unit["file"], [])]
|
||||
reason += "\n" + ("\n".join(dict.fromkeys(logs)) if logs else "frontend emitted no log lines mentioning labelled file")
|
||||
elif rows is None or shard_incomplete:
|
||||
status, reason = ("timeout" if backend.timed_out else "failed"), "question did not complete"
|
||||
elif not rows and rid not in backend.asked:
|
||||
status, reason = "failed", "query produced no markers"
|
||||
else:
|
||||
flows = [r for r in rows if isinstance(r, dict) and "kind" not in r and "sink" in r]
|
||||
if any(not row.get("trace") for row in flows):
|
||||
raise ValueError("taint witness returned without requested trace")
|
||||
extra.update(witness_rows=flows, traces=[parse_trace(r) for r in flows])
|
||||
status, reason = ("path" if flows else "asked-nothing"), ""
|
||||
record["units"][index] = unit_record(unit, status, reason, **extra)
|
||||
except (OSError, ValueError, RuntimeError) as exc:
|
||||
for index, unit in enumerate(pin["units"]):
|
||||
if unit["supported"]:
|
||||
record["units"][index] = unit_record(
|
||||
unit,
|
||||
"timeout" if backend.timed_out and not backend.fatal else "failed",
|
||||
str(exc),
|
||||
instrument=proof,
|
||||
questions={unit["unit"]: record["questions"][unit["unit"]]} if unit["unit"] in record.get("questions", {}) else {},
|
||||
witness_rows=[r for r in backend.answers.get(unit["unit"], []) if isinstance(r, dict) and "sink" in r],
|
||||
questions_completed=[unit["unit"]] if unit["unit"] in backend.answers else [],
|
||||
questions_asked=[unit["unit"]] if unit["unit"] in backend.asked else [],
|
||||
)
|
||||
finally:
|
||||
if hasattr(backend, "close_session"):
|
||||
backend.close_session()
|
||||
if cpg and cpg.cleanup:
|
||||
cpg.cleanup()
|
||||
for stage in backend.stages:
|
||||
if stage["command"] == "joern" and stage.get("exit") and "output_tail" in stage:
|
||||
detail = f"joern query exited {stage['exit']}: {stage['output_tail'] or 'no output'}"
|
||||
for row in record["units"]:
|
||||
if row["supported"] and row["status"] in {"failed", "timeout"}:
|
||||
row["reason"] = f"{row['reason']}\n{detail}".strip()
|
||||
record["questions_asked"] = list(dict.fromkeys(backend.asked))
|
||||
record["questions_completed"] = [rid for u in record["units"] for rid in u["questions_completed"]]
|
||||
record["done"] = True
|
||||
persist(out, record)
|
||||
return record
|
||||
|
||||
|
||||
def worker(pin, root, out, deadline, question_deadline, heap_profile="vm"):
|
||||
"""Run language partitions sequentially under one shared container deadline."""
|
||||
languages = sorted({"javascript" if u["language"] == "typescript" else u["language"] for u in pin["units"] if u["supported"]})
|
||||
if len(languages) <= 1:
|
||||
return worker_language(pin, root, out, deadline, question_deadline, heap_profile=heap_profile)
|
||||
end = time.monotonic() + deadline
|
||||
record = {
|
||||
**pin,
|
||||
"done": False,
|
||||
"units": [
|
||||
unit_record(u, "timeout" if u["supported"] else "unsupported", "pin deadline" if u["supported"] else u["reason"])
|
||||
for u in pin["units"]
|
||||
],
|
||||
"instruments": {},
|
||||
}
|
||||
|
||||
def persist(_, partial):
|
||||
updates = {u["unit"]: u for u in partial["units"]}
|
||||
record["units"] = [updates.get(u["unit"], u) for u in record["units"]]
|
||||
record["instruments"][language] = partial["instrument"]
|
||||
record["questions_asked"] = [rid for u in record["units"] for rid in u["questions_asked"]]
|
||||
record["questions_completed"] = [rid for u in record["units"] for rid in u["questions_completed"]]
|
||||
write_json(out, record)
|
||||
|
||||
write_json(out, record)
|
||||
for language in languages:
|
||||
remaining = end - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
units = [
|
||||
u for u in pin["units"] if u["supported"] and ("javascript" if u["language"] == "typescript" else u["language"]) == language
|
||||
]
|
||||
questions = {u["unit"]: pin["questions"][u["unit"]] for u in units}
|
||||
worker_language(
|
||||
{**pin, "units": units, "questions": questions}, root, out, remaining, question_deadline, persist, heap_profile=heap_profile
|
||||
)
|
||||
record["done"] = True
|
||||
write_json(out, record)
|
||||
return record
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--worker", type=Path, required=True)
|
||||
parser.add_argument("--deadline", type=int, default=900)
|
||||
parser.add_argument("--question-deadline", type=int, default=120)
|
||||
parser.add_argument("--heap-profile", choices=("cluster", "vm"), default="vm")
|
||||
parser.add_argument("--root", type=Path, default=Path("/case"))
|
||||
parser.add_argument("--output", type=Path, default=Path("/out/result.json"))
|
||||
args = parser.parse_args()
|
||||
if min(args.deadline, args.question_deadline) <= 0:
|
||||
parser.error("deadlines must be positive")
|
||||
worker(json.loads(args.worker.read_text()), args.root, args.output, args.deadline, args.question_deadline, args.heap_profile)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,315 +0,0 @@
|
||||
"""Image-resident queue workers. Requires strongly consistent S3 GET/PUT/LIST.
|
||||
|
||||
Per-task Lamport bakery tickets use only ordinary object operations. A choosing
|
||||
marker precedes ticket selection; contenders wait for choosing peers and lower
|
||||
(ticket, worker) pairs. Late arrivals cannot displace a running owner. Leases
|
||||
assume bounded clock skew (less than MARGIN / 2) and a worker killed by its deadline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import ctypes
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from engine_trace_worker import unit_record, worker
|
||||
|
||||
from openultrasast.plane.memory import S3Client, s3_settings
|
||||
|
||||
PENDING = "engine-queue/pending/"
|
||||
CLAIMED = "engine-queue/claimed/"
|
||||
RESULTS = "engine-results/"
|
||||
MARGIN = 60
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def client_from_env():
|
||||
settings = s3_settings(os.environ)
|
||||
bucket = os.environ.get("S3_BUCKET")
|
||||
if not bucket:
|
||||
raise ValueError("S3_BUCKET is required")
|
||||
return S3Client(bucket=bucket, **settings)
|
||||
|
||||
|
||||
def read(client, key):
|
||||
value = client.get(key)
|
||||
return json.loads(value[0]) if value else None
|
||||
|
||||
|
||||
def put(client, key, value):
|
||||
client.put(key, json.dumps(value).encode(), {})
|
||||
|
||||
|
||||
def result_key(task):
|
||||
return f"{RESULTS}{task['run']}/{task['id']}.json"
|
||||
|
||||
|
||||
def pack(root):
|
||||
stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=stream, mode="w") as archive:
|
||||
archive.add(root, arcname=".")
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def unpack(data, root):
|
||||
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
||||
# Source snapshots need only regular files and directories, never links/devices.
|
||||
for member in archive.getmembers():
|
||||
target = (root / member.name).resolve()
|
||||
if not target.is_relative_to(root.resolve()) or not (member.isfile() or member.isdir()):
|
||||
raise ValueError("unsafe source archive")
|
||||
archive.extractall(root, filter="data")
|
||||
|
||||
|
||||
class Queue:
|
||||
def __init__(self, client, identity=None, *, now=time.time, pause=time.sleep, stop=None):
|
||||
self.client = client
|
||||
self.identity = identity or f"{socket.gethostname()}-{uuid.uuid4().hex}"
|
||||
self.now, self.pause = now, pause
|
||||
self.stop = stop or threading.Event()
|
||||
|
||||
def markers(self, task_id=None):
|
||||
result = []
|
||||
for key in self.client.keys(CLAIMED):
|
||||
if task_id is None or key.rsplit("/", 1)[-1] == task_id:
|
||||
marker = read(self.client, key)
|
||||
if marker:
|
||||
result.append((key, marker))
|
||||
return result
|
||||
|
||||
def tasks(self):
|
||||
tasks = {}
|
||||
for key in self.client.keys(PENDING):
|
||||
task = read(self.client, key)
|
||||
if task:
|
||||
tasks[task["task_id"]] = task
|
||||
# A dead worker may have removed pending already. Its task remains in the lease.
|
||||
for _, marker in self.markers():
|
||||
if marker["expires"] < self.now():
|
||||
task = marker["task"]
|
||||
tasks.setdefault(task["task_id"], task)
|
||||
return list(tasks.values())
|
||||
|
||||
def claim(self, task):
|
||||
if read(self.client, result_key(task)) is not None or self.stop.is_set():
|
||||
return None
|
||||
key = f"{CLAIMED}{self.identity}/{task['task_id']}"
|
||||
marker = dict(task=task, ticket=0, expires=self.now() + task["deadline"] + MARGIN)
|
||||
put(self.client, key, marker)
|
||||
try:
|
||||
live = [m for _, m in self.markers(task["task_id"]) if m["expires"] > self.now()]
|
||||
marker["ticket"] = 1 + max((m["ticket"] for m in live), default=0)
|
||||
put(self.client, key, marker)
|
||||
while not self.stop.is_set():
|
||||
# Do not start once the execution budget no longer fits within our lease.
|
||||
if self.now() >= marker["expires"] - task["deadline"] - MARGIN / 2:
|
||||
break
|
||||
peers = [(k, m) for k, m in self.markers(task["task_id"]) if k != key and m["expires"] > self.now()]
|
||||
if any(m["ticket"] == 0 for _, m in peers):
|
||||
self.pause(0.1)
|
||||
continue
|
||||
if any((m["ticket"], k) < (marker["ticket"], key) for k, m in peers):
|
||||
break # loser backs off and can try another pending task
|
||||
if read(self.client, key) != marker or read(self.client, result_key(task)) is not None:
|
||||
break
|
||||
self.client.delete(PENDING + task["task_id"])
|
||||
return key
|
||||
except Exception:
|
||||
self.client.delete(key)
|
||||
raise
|
||||
self.client.delete(key)
|
||||
return None
|
||||
|
||||
def process_one(self, execute=None):
|
||||
for task in self.tasks():
|
||||
key = self.claim(task)
|
||||
if key is None:
|
||||
continue
|
||||
try:
|
||||
lease = read(self.client, key)
|
||||
if lease is None:
|
||||
continue
|
||||
budget = min(task["deadline"], lease["expires"] - self.now() - MARGIN / 2)
|
||||
if budget <= 0:
|
||||
continue # Leave the expired lease's task available to recovery.
|
||||
result = (execute or execute_task)(self.client, {**task, "runtime_budget": budget})
|
||||
result.update(executor="queue", worker=self.identity, pod=socket.gethostname(), image=os.environ.get("ENGINE_IMAGE", ""))
|
||||
put(self.client, result_key(task), result)
|
||||
# Publish the result before removing either the input or the claim.
|
||||
self.client.delete(task["input"])
|
||||
for old_key, old in self.markers(task["task_id"]):
|
||||
if old_key == key or old["expires"] < self.now():
|
||||
self.client.delete(old_key)
|
||||
return True
|
||||
except Exception:
|
||||
# Keep the lease and task for recovery, without logging credential-bearing errors.
|
||||
LOG.error("task failed; lease retained for recovery")
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _child(pin, root, out, deadline, question_deadline):
|
||||
os.setsid()
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
worker(pin, root, out, deadline, question_deadline)
|
||||
|
||||
|
||||
def adopt_orphans():
|
||||
# Linux PR_SET_CHILD_SUBREAPER: escaped Joern sessions reparent here, not
|
||||
# outside the service. There is exactly one task process tree per worker.
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError("cannot enable task descendant cleanup")
|
||||
|
||||
|
||||
def kill_children():
|
||||
"""Freeze parents before walking children, including new-session JVMs."""
|
||||
seen = set()
|
||||
|
||||
def freeze(parent):
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdigit() or int(entry.name) in seen:
|
||||
continue
|
||||
try:
|
||||
# comm can contain spaces or parentheses. Fields after it begin at state.
|
||||
fields = (entry / "stat").read_text().rsplit(")", 1)[1].split()
|
||||
if int(fields[1]) != parent:
|
||||
continue
|
||||
pid = int(entry.name)
|
||||
os.kill(pid, signal.SIGSTOP)
|
||||
seen.add(pid)
|
||||
freeze(pid)
|
||||
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
||||
continue
|
||||
|
||||
# A child that exited during discovery can reparent descendants to this
|
||||
# subreaper. Repeat until every remaining descendant is frozen.
|
||||
while True:
|
||||
count = len(seen)
|
||||
freeze(os.getpid())
|
||||
if len(seen) == count:
|
||||
break
|
||||
for pid in seen:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
|
||||
|
||||
def execute_task(client, task):
|
||||
adopt_orphans()
|
||||
pin = task["pin"]
|
||||
started = time.monotonic()
|
||||
budget = task.get("runtime_budget", task["deadline"])
|
||||
status, reason, code = "failed", "worker produced no result", None
|
||||
with tempfile.TemporaryDirectory(prefix="engine-task-") as scratch:
|
||||
root, out = Path(scratch) / "case", Path(scratch) / "result.json"
|
||||
root.mkdir()
|
||||
try:
|
||||
if task["deadline"] > 900 or task["deadline"] <= 0:
|
||||
raise ValueError("deadline exceeds deployment grace")
|
||||
if task.get("image") != os.environ.get("ENGINE_IMAGE"):
|
||||
raise ValueError("task image differs from deployed image")
|
||||
source = client.get(task["input"])
|
||||
if source is None:
|
||||
raise ValueError("missing source")
|
||||
unpack(source[0], root)
|
||||
# The shipped worker also checks frontend input and graph output for every language.
|
||||
files = [p for p in root.rglob("*") if p.is_file()]
|
||||
if not files:
|
||||
raise ValueError("empty source")
|
||||
LOG.info("input proof: first file bytes=%d", len(files[0].read_bytes()))
|
||||
if time.monotonic() - started >= budget:
|
||||
status, reason = "timeout", "task input deadline"
|
||||
raise TimeoutError(reason)
|
||||
process = multiprocessing.get_context("fork").Process(target=_child, args=(pin, root, out, budget, task["question_deadline"]))
|
||||
process.start()
|
||||
try:
|
||||
process.join(max(0, budget - (time.monotonic() - started)))
|
||||
if process.is_alive():
|
||||
status, reason = "timeout", "task deadline"
|
||||
else:
|
||||
reason = f"worker exit {process.exitcode}"
|
||||
code = process.exitcode
|
||||
finally:
|
||||
# Joern creates separate sessions. Kill the whole descendant tree,
|
||||
# not merely the Python wrapper's process group.
|
||||
kill_children()
|
||||
process.join()
|
||||
# Reap adopted JVMs and their launchers after the direct child.
|
||||
while True:
|
||||
try:
|
||||
os.waitpid(-1, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
record = json.loads(out.read_text()) if out.exists() else {**pin, "units": []}
|
||||
except Exception:
|
||||
record = {**pin, "units": []}
|
||||
reason = "worker input or execution failed"
|
||||
if not record.get("done") or code != 0:
|
||||
previous = {u["unit"]: u for u in record["units"]}
|
||||
record["units"] = [
|
||||
previous[u["unit"]]
|
||||
if code is None and previous.get(u["unit"], {}).get("status") in {"path", "asked-nothing"}
|
||||
else unit_record(u, status if u["supported"] else "unsupported", reason if u["supported"] else u["reason"])
|
||||
for u in pin["units"]
|
||||
]
|
||||
record.update(done=True, container_exit=code, seconds=time.monotonic() - started)
|
||||
return record
|
||||
|
||||
|
||||
def status(client):
|
||||
return {
|
||||
"pending": len(client.keys(PENDING)),
|
||||
"claimed": len(client.keys(CLAIMED)),
|
||||
"done": len([k for k in client.keys(RESULTS) if k.endswith(".json")]),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
stop = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: stop.set())
|
||||
client = client_from_env()
|
||||
queue = Queue(client, stop=stop)
|
||||
|
||||
class Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200 if self.path == "/healthz" and not stop.is_set() else 503)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("0.0.0.0", args.port), Health)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
if not queue.process_one():
|
||||
stop.wait(2)
|
||||
except Exception:
|
||||
LOG.error("queue unavailable; retrying")
|
||||
stop.wait(5)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,239 +0,0 @@
|
||||
"""Offline audit of completed Joern results; never launches an analyzer or reads corpora.
|
||||
|
||||
Missing results stay in coverage, not signal denominators. Observed non-path rows
|
||||
score zero for path features. Sink kinds are one-hot indicators, never ordinal
|
||||
category IDs. Both signed and best-direction metrics expose reversed shortcuts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from slice_leak_audit import THRESHOLD, complete_pairs, metrics
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
UNITS = ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl"
|
||||
RESULTS = Path.home() / "ousast-results/plane/engine-trace"
|
||||
OUTPUT = ROOT / "benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json"
|
||||
STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported", "missing")
|
||||
ASKED = {"path", "asked-nothing"}
|
||||
SIGNALS = ("has_path", "shortest_path_steps", "sanitized", "guard_present", "asked")
|
||||
ASYMMETRY = ("vulnerable_only", "fixed_only", "both", "neither", "not_both_asked")
|
||||
|
||||
|
||||
def materialization_version():
|
||||
# Read the producer's literal without importing its execution dependencies.
|
||||
tree = ast.parse(Path(__file__).with_name("engine_trace.py").read_bytes())
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "MATERIALIZATION_VERSION" for t in node.targets):
|
||||
return ast.literal_eval(node.value)
|
||||
raise ValueError("Producer has no materialization version")
|
||||
|
||||
|
||||
def features(row):
|
||||
path = row["status"] == "path"
|
||||
traces = row.get("traces", []) if path else []
|
||||
candidates = [(i, t) for i, t in enumerate(traces) if t.get("steps")]
|
||||
index, trace = min(candidates, key=lambda it: len(it[1]["steps"])) if candidates else (None, {})
|
||||
witnesses = row.get("witness_rows", [])
|
||||
witness = witnesses[index] if index is not None and index < len(witnesses) else {}
|
||||
guard = trace.get("guard", {})
|
||||
sink = witness.get("sinkKind")
|
||||
if not sink:
|
||||
# Retain only the call target: arguments, literals and code are not kinds.
|
||||
match = re.search(r"([\w:$>.\\-]+)\s*\(", str(witness.get("sink", "")))
|
||||
sink = match[1] if match else "unknown"
|
||||
return {
|
||||
"has_path": int(path),
|
||||
"shortest_path_steps": len(trace["steps"]) if trace else None if path else 0,
|
||||
"sanitized": int(bool(trace.get("sanitized", witness.get("sanitized", False)))) if trace else None if path else 0,
|
||||
"guard_present": int(
|
||||
bool(guard.get("text") or guard.get("bounded") or witness.get("bound") or witness.get("bounded"))
|
||||
or any("guard" in s.get("roles", []) for s in trace.get("steps", []))
|
||||
)
|
||||
if trace
|
||||
else None
|
||||
if path
|
||||
else 0,
|
||||
"sink_kind": str(sink) if path else "no-path",
|
||||
"asked": int(row["status"] in ASKED),
|
||||
}
|
||||
|
||||
|
||||
def scored(rows, signal):
|
||||
eligible = [r for r in rows if r[signal] is not None]
|
||||
result = metrics(eligible, signal)
|
||||
result["excluded_rows"] = len(rows) - len(eligible)
|
||||
result["flagged"] = (result["pairs"] >= 10 and (result["best_direction_within_pair_accuracy"] or 0) >= THRESHOLD) or (
|
||||
result["rows"] >= 10 and (result["best_direction_pooled_auc"] or 0) >= THRESHOLD
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def signal_metrics(rows, kinds):
|
||||
return {
|
||||
**{s: scored(rows, s) for s in SIGNALS},
|
||||
"sink_kind": {kind: scored([{**r, "is_kind": int(r["sink_kind"] == kind)} for r in rows], "is_kind") for kind in kinds},
|
||||
}
|
||||
|
||||
|
||||
def scope_report(rows, kinds):
|
||||
pairs = complete_pairs(rows)
|
||||
asymmetry = Counter()
|
||||
honest = []
|
||||
for v, f in pairs:
|
||||
if not (v["asked"] and f["asked"]):
|
||||
asymmetry["not_both_asked"] += 1
|
||||
else:
|
||||
honest.extend((v, f))
|
||||
asymmetry[
|
||||
"both"
|
||||
if v["has_path"] and f["has_path"]
|
||||
else "vulnerable_only"
|
||||
if v["has_path"]
|
||||
else "fixed_only"
|
||||
if f["has_path"]
|
||||
else "neither"
|
||||
] += 1
|
||||
observed = [r for r in rows if r["status"] != "missing"]
|
||||
return {
|
||||
"rows": len(rows),
|
||||
"coverage": {s: sum(r["status"] == s for r in rows) for s in STATUSES},
|
||||
"complete_pairs": len(pairs),
|
||||
"pair_asymmetry": {s: asymmetry[s] for s in ASYMMETRY},
|
||||
"both_asked_pairs": len(honest) // 2,
|
||||
"signals": signal_metrics(observed, kinds),
|
||||
"both_asked_signals": signal_metrics(honest, kinds),
|
||||
}
|
||||
|
||||
|
||||
def audit(units, results, *, expected_rows=501, expected_pairs=195):
|
||||
if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
|
||||
raise ValueError(f"Expected {expected_rows} unique units")
|
||||
if len(complete_pairs(units)) != expected_pairs:
|
||||
raise ValueError(f"Expected {expected_pairs} complete pairs")
|
||||
if not results.is_dir():
|
||||
raise ValueError(f"Results directory is unreadable: {results}")
|
||||
version = materialization_version()
|
||||
unit_index = {u["unit"]: u for u in units}
|
||||
matched = {}
|
||||
files = []
|
||||
counts = Counter()
|
||||
for path in sorted(results.glob("*.json")):
|
||||
raw = path.read_bytes()
|
||||
record = json.loads(raw) # Corruption fails loudly; producer uses atomic replace.
|
||||
if not isinstance(record, dict) or "repo" not in record or "pin" not in record:
|
||||
continue
|
||||
stale = record.get("materialization_version") != version
|
||||
unfinished = record.get("done") is not True
|
||||
counts["stale"] += stale
|
||||
counts["unfinished"] += unfinished
|
||||
counts["skipped"] += stale or unfinished
|
||||
files.append((path.name, len(raw)))
|
||||
if stale or unfinished:
|
||||
continue
|
||||
counts["accepted_files"] += 1
|
||||
counts["result_rows_read"] += len(record["units"])
|
||||
for row in record["units"]:
|
||||
if row["unit"] not in unit_index:
|
||||
continue
|
||||
unit = unit_index[row["unit"]]
|
||||
if any(row.get(k) != unit.get(k) for k in ("family", "label", "pair")):
|
||||
raise ValueError(f"Result metadata mismatch: {row['unit']}")
|
||||
if row["status"] not in STATUSES[:-1] or not row.get("source"):
|
||||
raise ValueError(f"Invalid result status/source: {row['unit']}")
|
||||
if row["unit"] in matched:
|
||||
raise ValueError(f"Duplicate current result: {row['unit']}")
|
||||
matched[row["unit"]] = {
|
||||
**unit,
|
||||
"source": row["source"],
|
||||
"status": row["status"],
|
||||
"pin_role": row.get("pin_role"),
|
||||
**features(row),
|
||||
}
|
||||
rows = [
|
||||
matched.get(u["unit"], {**u, "source": u.get("source", "unknown"), "status": "missing", **features({"status": "missing"})})
|
||||
for u in units
|
||||
]
|
||||
kinds = sorted({r["sink_kind"] for r in rows if r["status"] != "missing"})
|
||||
scopes = {"overall": scope_report(rows, kinds)}
|
||||
for field in ("family", "source"):
|
||||
scopes[f"by_{field}"] = {
|
||||
value: scope_report([r for r in rows if r[field] == value], kinds) for value in sorted({r[field] for r in rows})
|
||||
}
|
||||
return {
|
||||
"cost_usd": 0,
|
||||
"materialization_version": version,
|
||||
"threshold": THRESHOLD,
|
||||
"minimum_samples": 10,
|
||||
"unit_rows_read": len(units),
|
||||
"matched_rows": len(matched),
|
||||
"inputs": {
|
||||
**{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
|
||||
"files_read": len(files),
|
||||
"bytes_read": sum(size for _, size in files),
|
||||
# Hash a canonical inventory; filenames and paths may identify repositories.
|
||||
"files_sha256": hashlib.sha256(json.dumps(sorted(files), separators=(",", ":")).encode()).hexdigest(),
|
||||
},
|
||||
"semantics": {
|
||||
"signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
|
||||
"missing": "Coverage only; absent source metadata is unknown. Skipped result units are never consumed.",
|
||||
"non_path": "Observed non-path rows score zero; path rows lacking trace evidence have null trace features.",
|
||||
"asked": "path or asked-nothing=1; failed, timeout, unsupported=0; missing excluded from metrics.",
|
||||
"asymmetry": "Disjoint categories; not_both_asked takes precedence over path asymmetry.",
|
||||
"flags": "Best of both directions, >=0.70 with >=10 pairs for pair accuracy or >=10 rows for AUC.",
|
||||
"sink_kind": "One binary indicator per observed call target, including no-path/unknown; no ordinal encoding.",
|
||||
"limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
|
||||
},
|
||||
**scopes,
|
||||
"units": [{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", *SIGNALS, "sink_kind")} for r in rows],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--results", type=Path, default=RESULTS)
|
||||
parser.add_argument("--units", type=Path, default=UNITS)
|
||||
parser.add_argument("--output", type=Path, default=OUTPUT)
|
||||
args = parser.parse_args()
|
||||
raw = args.units.read_bytes()
|
||||
units = [json.loads(line) for line in raw.splitlines() if line.strip()]
|
||||
report = audit(units, args.results)
|
||||
report["units_input"] = {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
overall, inputs = report["overall"], report["inputs"]
|
||||
lines = [
|
||||
f"Joern leak audit: $0; materialization_version={report['materialization_version']}",
|
||||
f"Units read: {len(units)}; bytes={len(raw)}; file={args.units}",
|
||||
f"Result files read: {inputs['files_read']}; bytes={inputs['bytes_read']}; accepted={inputs['accepted_files']}",
|
||||
f"Skipped: {inputs['skipped']} (stale={inputs['stale']}, unfinished={inputs['unfinished']}; counts overlap)",
|
||||
f"Result rows read: {inputs['result_rows_read']}; matched={report['matched_rows']}",
|
||||
f"Coverage: {overall['coverage']}",
|
||||
f"Pairs: {overall['complete_pairs']}; both asked={overall['both_asked_pairs']}",
|
||||
f"Asymmetry: {overall['pair_asymmetry']}",
|
||||
]
|
||||
for signal in (*SIGNALS, "sink_kind"):
|
||||
|
||||
def brief(value, signal=signal):
|
||||
if signal == "sink_kind":
|
||||
return {k: brief_metric(m) for k, m in value.items()}
|
||||
return brief_metric(value)
|
||||
|
||||
lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
|
||||
lines.append(f"Family/source metrics and input inventory digest: {args.output}")
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
def brief_metric(metric):
|
||||
return {k: metric[k] for k in ("rows", "pairs", "within_pair_accuracy", "pooled_auc", "flagged")}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -141,9 +141,7 @@ def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_ro
|
||||
"limitations": (
|
||||
"Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
|
||||
),
|
||||
"units": [
|
||||
{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", "has_flow", "slice_length")} for r in rows
|
||||
],
|
||||
"units": rows,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"modules": 172,
|
||||
"modules": 159,
|
||||
"counts": {
|
||||
"load_bearing": 166,
|
||||
"standalone_capability": 6
|
||||
"load_bearing": 154,
|
||||
"standalone_capability": 5
|
||||
},
|
||||
"orphaned": [],
|
||||
"standalone": {
|
||||
@@ -10,7 +10,6 @@
|
||||
"mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration",
|
||||
"push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs",
|
||||
"push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring",
|
||||
"search.coordinator": "search-with-proof Req 1/2/4: injected search executor and offline board checkpoint inspector; agent worker pending",
|
||||
"skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path"
|
||||
},
|
||||
"rows": [
|
||||
@@ -638,7 +637,6 @@
|
||||
"push.runner",
|
||||
"push.snapshot",
|
||||
"push.vetoes",
|
||||
"search.worker",
|
||||
"tool_hunter"
|
||||
]
|
||||
},
|
||||
@@ -709,7 +707,6 @@
|
||||
"plane.budget",
|
||||
"plane.generate",
|
||||
"push.assistance",
|
||||
"search.worker",
|
||||
"tool_hunter"
|
||||
]
|
||||
},
|
||||
@@ -900,10 +897,7 @@
|
||||
"plane.tasks.remember",
|
||||
"plane.tasks.roles",
|
||||
"plane.tasks.verify",
|
||||
"plane.workspaces",
|
||||
"search.board",
|
||||
"search.coordinator",
|
||||
"search.worker"
|
||||
"plane.workspaces"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -919,9 +913,7 @@
|
||||
"learn.program",
|
||||
"plane.generate",
|
||||
"plane.tasks.roles",
|
||||
"plane.tasks.verify",
|
||||
"search.coordinator",
|
||||
"search.worker"
|
||||
"plane.tasks.verify"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1008,9 +1000,7 @@
|
||||
"plane.generate",
|
||||
"plane.reconciler",
|
||||
"plane.tasks.loop",
|
||||
"plane.tasks.remember",
|
||||
"search.board",
|
||||
"search.coordinator"
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1239,7 +1229,6 @@
|
||||
"learn.compile",
|
||||
"learn.program",
|
||||
"model.endpoint",
|
||||
"search.worker",
|
||||
"tool_hunter"
|
||||
]
|
||||
},
|
||||
@@ -1252,7 +1241,6 @@
|
||||
"learn.compile",
|
||||
"learn.program",
|
||||
"model.endpoint",
|
||||
"search.worker",
|
||||
"tool_hunter"
|
||||
]
|
||||
},
|
||||
@@ -1565,10 +1553,7 @@
|
||||
"regress.candidate",
|
||||
"regress.recipes",
|
||||
"regress.safety",
|
||||
"regress.verdict",
|
||||
"search._sandbox",
|
||||
"search.executor",
|
||||
"search.verify"
|
||||
"regress.verdict"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1607,125 +1592,6 @@
|
||||
"scoring"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.coordinator",
|
||||
"search.executor",
|
||||
"search.oracles",
|
||||
"search.probe",
|
||||
"search.verify",
|
||||
"search.verify_task",
|
||||
"search.worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search._sandbox",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.executor",
|
||||
"search.oracles",
|
||||
"search.probe",
|
||||
"search.verify",
|
||||
"search.worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.board",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.coordinator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.budget",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.coordinator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.coordinator",
|
||||
"classification": "standalone_capability",
|
||||
"reason": "search-with-proof Req 1/2/4: injected search executor and offline board checkpoint inspector; agent worker pending",
|
||||
"importers": [
|
||||
"search.worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.demo",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.executor",
|
||||
"search.verify",
|
||||
"search.verify_task",
|
||||
"search.worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.executor",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.executor",
|
||||
"importers": [
|
||||
"search.verify_task",
|
||||
"search.worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.oracles",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.oracles",
|
||||
"importers": [
|
||||
"search.probe",
|
||||
"search.verify"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.probe",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.probe",
|
||||
"importers": []
|
||||
},
|
||||
{
|
||||
"module": "search.task_storage",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate, cli",
|
||||
"importers": [
|
||||
"search.executor",
|
||||
"search.verify",
|
||||
"search.verify_task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.verify",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.verify",
|
||||
"importers": [
|
||||
"search.executor",
|
||||
"search.probe",
|
||||
"search.verify_task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.verify_task",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.verify_task",
|
||||
"importers": [
|
||||
"search.executor"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "search.worker",
|
||||
"classification": "load_bearing",
|
||||
"reason": "python -m openultrasast.search.worker",
|
||||
"importers": []
|
||||
},
|
||||
{
|
||||
"module": "semantic",
|
||||
"classification": "load_bearing",
|
||||
|
||||
@@ -1,326 +0,0 @@
|
||||
{
|
||||
"created": "2026-10-04",
|
||||
"note": "COUNTS ONLY. Batch 3 of real (vulnerable function, fixed function) pairs whose function survives the fix, mined at zero model cost from reviewed GitHub advisories (pip, npm, maven, composer) and added as pointer-pair corpus benchmarks/pairs/advisory-fixes-3 (source `advisory-fixes-3` in learn/sources.toml); labelled, given static feature records, built into the memory store and embedded. No chat-model call, no Docker, no Joern. Candidate, path, function and repository names stay in the catalog; the pairs the reserved-population guard flagged are withheld (found by bisection with the test output discarded; names never printed).",
|
||||
"method": {
|
||||
"enumeration": "gh api /advisories?type=reviewed&ecosystem=<eco>&cwes=<family CWEs>&published=>2023-01-01 per ecosystem (pip, npm, maven, composer) x six compiled families; Go, Ruby and C# not enumerated; newest first",
|
||||
"advisories_enumerated": 8531,
|
||||
"advisories_published_after_2026_06_01": 1855,
|
||||
"automated_screen": "as batch 2 (single family by CWE; linked fix commit, at most 3, one repository; repository not named anywhere under benchmarks/ incl. batches 1-2; excluded v3 file names never opened; no WordPress plugin; one pair per repository; SPDX licence permissive; one-parent commit; the function holding most changed code lines inside its own matcher span, declared once on both sides, body changed, >= 25% of the file's changed code lines) plus three widenings: (a) an advisory linking only a merged pull request uses its single commit or its squash/rebase merge commit; (b) commits of up to 6 non-test source files / 240 changed lines (batch 2: 4 / 120); (c) a second pass over advisories whose CWEs span two families, the family then chosen from the function's diff on review",
|
||||
"file_reads": "commit metadata and the two versions of each changed source file through the contents API into scratch (no clone, no scanner, engine, plane Run or model on these repositories); scratch deleted after",
|
||||
"manual_review": "every automated candidate's in-function diff read (agent, no model call); kept only when the named function itself holds the fix; a weak inline check replaced by a stronger helper call counts as the fix, a function that only forwards a flag or calls a fix made elsewhere does not",
|
||||
"reserved_population_guard": "pytest -q tests/ -k 'reserved or v3' with output discarded, bisected over catalog subsets (18 runs; baseline with an empty batch green); final catalog green",
|
||||
"rename_check": "canonical full_name of every kept repository from the API: no two kept pairs share a repository, none renamed into a name the benchmarks use",
|
||||
"mechanism": "family default as batch 2 (injection/output_encoding/deserialization source_reaches_sink, path path_join_user_input, access_control missing_auth_guard, untrusted_destination validation_strength for CWE-601 else source_reaches_sink); provenance = agent"
|
||||
},
|
||||
"screened": {
|
||||
"advisories_examined": 8805,
|
||||
"outcomes": {
|
||||
"no fix commit linked": 3480,
|
||||
"repository already in benchmarks/ (populations v1/v2 or a pair corpus)": 1993,
|
||||
"licence not permissive or unrecorded": 1249,
|
||||
"repository already kept (one pair per repository)": 657,
|
||||
"kept": 398,
|
||||
"CWEs span several families": 274,
|
||||
"no function declared on both sides with a changed body": 191,
|
||||
"large commit (refactor or release, not the fix)": 173,
|
||||
"fix spread over several commits or repositories": 167,
|
||||
"merge or root commit": 115,
|
||||
"no source file changed": 42,
|
||||
"language outside the feature vocabulary (go/ruby/c#/...)": 32,
|
||||
"manual review: the chosen function does not hold the fix (comment-only, misattributed, partial or family mismatch)": 19,
|
||||
"repository unavailable": 9,
|
||||
"WordPress plugin (population v3 safety)": 4,
|
||||
"fix commit unavailable": 2
|
||||
},
|
||||
"outcomes_note": "single-family pass (screen 2, every advisory examined, no quota met) plus the two-family pass; `kept` = automated candidates; `manual review: the chosen function does not hold the fix (comment-only, misattributed, partial or family mismatch)` = the screen re-meeting a function the 2026-10-03 review had dropped",
|
||||
"candidates_manually_reviewed": 412,
|
||||
"kept_after_review": 255,
|
||||
"review_drops": {
|
||||
"total": 157,
|
||||
"change in the function unrelated to the advisory (refactor, rename, formatting, another advisory)": 33,
|
||||
"function misattributed (matcher span overrun or an undetected enclosing declaration)": 33,
|
||||
"partial: the function only calls or configures the real fix": 29,
|
||||
"function diff not small (rewrite or refactor around the fix)": 28,
|
||||
"unsure": 17,
|
||||
"family mismatch": 11,
|
||||
"same fix as a kept pair in a fork or sibling repository (would leak across repository folds)": 3,
|
||||
"WordPress plugin (population v3 safety; missed by the automated screen)": 2,
|
||||
"reserved-population guard (flagged by the 2026-10-03 guard run; name not printed)": 1
|
||||
},
|
||||
"one_pair_per_repository_drops": 3,
|
||||
"reserved_population_drops": 2,
|
||||
"kept": 251
|
||||
},
|
||||
"kept": {
|
||||
"total": 251,
|
||||
"by_family": {
|
||||
"access_control": 20,
|
||||
"deserialization": 9,
|
||||
"injection": 56,
|
||||
"output_encoding": 69,
|
||||
"path": 71,
|
||||
"untrusted_destination": 26
|
||||
},
|
||||
"post_cutoff_by_family": {
|
||||
"injection": 5,
|
||||
"output_encoding": 5,
|
||||
"path": 2
|
||||
},
|
||||
"post_cutoff_total": 12,
|
||||
"by_language": {
|
||||
"java": 37,
|
||||
"javascript": 37,
|
||||
"php": 46,
|
||||
"python": 83,
|
||||
"typescript": 48
|
||||
},
|
||||
"post_cutoff_by_language": {
|
||||
"python": 6,
|
||||
"typescript": 6
|
||||
},
|
||||
"by_family_language": {
|
||||
"access_control:java": 5,
|
||||
"access_control:javascript": 3,
|
||||
"access_control:php": 7,
|
||||
"access_control:python": 2,
|
||||
"access_control:typescript": 3,
|
||||
"deserialization:java": 2,
|
||||
"deserialization:php": 1,
|
||||
"deserialization:python": 6,
|
||||
"injection:java": 5,
|
||||
"injection:javascript": 7,
|
||||
"injection:php": 9,
|
||||
"injection:python": 24,
|
||||
"injection:typescript": 11,
|
||||
"output_encoding:java": 7,
|
||||
"output_encoding:javascript": 20,
|
||||
"output_encoding:php": 12,
|
||||
"output_encoding:python": 14,
|
||||
"output_encoding:typescript": 16,
|
||||
"path:java": 10,
|
||||
"path:javascript": 4,
|
||||
"path:php": 10,
|
||||
"path:python": 34,
|
||||
"path:typescript": 13,
|
||||
"untrusted_destination:java": 8,
|
||||
"untrusted_destination:javascript": 3,
|
||||
"untrusted_destination:php": 7,
|
||||
"untrusted_destination:python": 3,
|
||||
"untrusted_destination:typescript": 5
|
||||
},
|
||||
"by_ecosystem": {
|
||||
"composer": 46,
|
||||
"maven": 37,
|
||||
"npm": 81,
|
||||
"pip": 87
|
||||
},
|
||||
"licences": {
|
||||
"Apache-2.0": 80,
|
||||
"BSD-2-Clause": 3,
|
||||
"BSD-3-Clause": 18,
|
||||
"ISC": 1,
|
||||
"MIT": 147,
|
||||
"MPL-2.0": 1,
|
||||
"Unlicense": 1
|
||||
},
|
||||
"split": {
|
||||
"holdout": 124,
|
||||
"train": 127
|
||||
},
|
||||
"published_by_year": {
|
||||
"2023": 50,
|
||||
"2024": 52,
|
||||
"2025": 66,
|
||||
"2026": 83
|
||||
},
|
||||
"published_range": [
|
||||
"2023-01-04",
|
||||
"2026-10-01"
|
||||
],
|
||||
"via_pull_request": 13,
|
||||
"commit_over_120_source_lines": 10,
|
||||
"two_family_advisory_family_chosen_on_review": 3,
|
||||
"verification": {
|
||||
"verified": 251,
|
||||
"excerpt_bytes_read": 1297516
|
||||
},
|
||||
"targets": {
|
||||
"asked": "injection and untrusted_destination >= 60 each; path, access_control, output_encoding, deserialization >= 30 where advisories allow; >= 250 total",
|
||||
"met": {
|
||||
"access_control": false,
|
||||
"deserialization": false,
|
||||
"injection": false,
|
||||
"output_encoding": true,
|
||||
"path": true,
|
||||
"untrusted_destination": false
|
||||
},
|
||||
"limit": "the reviewed-advisory pool, not the review: every advisory since 2023-01-01 in the four ecosystems was examined; the post-cutoff pool was largely spent by batch 2 (its repositories are now excluded), and untrusted_destination/access_control/deserialization advisories mostly lack a linked single fix commit or sit in repositories the benchmarks already use"
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"command": "ousast learn labels --root . --out <scratch> --snapshot <scratch>",
|
||||
"rows": [
|
||||
6981,
|
||||
7985
|
||||
],
|
||||
"pairs_rows": [
|
||||
1936,
|
||||
2940
|
||||
],
|
||||
"new_pair_rows": 1004
|
||||
},
|
||||
"features": {
|
||||
"command": "benchmarks/learn/build_features.py --labels <the 1004 new rows> --units <merged units.json of `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes-3/catalog.toml` (manifests written, no Run started)> --catalog benchmarks/pairs/advisory-fixes-3/catalog.toml --kinds pairs",
|
||||
"records": {
|
||||
"plane:access_control": 40,
|
||||
"plane:deserialization": 18,
|
||||
"plane:injection": 112,
|
||||
"plane:output_encoding": 138,
|
||||
"plane:path": 142,
|
||||
"plane:untrusted_destination": 52,
|
||||
"static:access_control": 40,
|
||||
"static:deserialization": 18,
|
||||
"static:injection": 112,
|
||||
"static:output_encoding": 138,
|
||||
"static:path": 142,
|
||||
"static:untrusted_destination": 52
|
||||
},
|
||||
"instrument_states_plane_all_families": {
|
||||
"agree:missing": 164,
|
||||
"agree:none": 338,
|
||||
"delta:not_applicable": 502,
|
||||
"engine:missing": 406,
|
||||
"engine:none": 96,
|
||||
"entry_points:not_applicable": 502,
|
||||
"facts:not_applicable": 502,
|
||||
"language:ran": 502,
|
||||
"model_sinks:missing": 502,
|
||||
"quick:ran": 502,
|
||||
"roles:ran": 502,
|
||||
"source:ran": 502,
|
||||
"verify:missing": 164,
|
||||
"verify:none": 338
|
||||
},
|
||||
"note": "quick rules and deterministic roles ran on each side; engine, verify, agree and model sinks never ran for these pairs (missing/none: null features, never zeros)"
|
||||
},
|
||||
"memory_build": {
|
||||
"command": "ousast learn memory build --labels <labels> --units <merged units.json> --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)",
|
||||
"examples": [
|
||||
1133,
|
||||
1635
|
||||
],
|
||||
"pairs_examples": [
|
||||
968,
|
||||
1470
|
||||
],
|
||||
"pairs_counts_after": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 1470,
|
||||
"labels": 2940,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 1470,
|
||||
"recorded": 1470,
|
||||
"unread": 0
|
||||
}
|
||||
},
|
||||
"complete_pairs": {
|
||||
"how": "store example rows (profile static) keyed as learn.experiments.freeze_units pairs them: (repo, candidate, family) holding both labels; whole memory, every family",
|
||||
"before": 459,
|
||||
"after": 710,
|
||||
"by_family": {
|
||||
"access_control": [
|
||||
37,
|
||||
57
|
||||
],
|
||||
"config_secrets": [
|
||||
12,
|
||||
12
|
||||
],
|
||||
"deserialization": [
|
||||
32,
|
||||
41
|
||||
],
|
||||
"injection": [
|
||||
68,
|
||||
124
|
||||
],
|
||||
"memory": [
|
||||
149,
|
||||
149
|
||||
],
|
||||
"output_encoding": [
|
||||
77,
|
||||
146
|
||||
],
|
||||
"path": [
|
||||
38,
|
||||
109
|
||||
],
|
||||
"unknown": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"untrusted_destination": [
|
||||
44,
|
||||
70
|
||||
]
|
||||
},
|
||||
"exp_004_005_units": "195 complete pairs inside exp-004/005's frozen 501 units (six families, evaluation folds); those units are frozen and were not regenerated here"
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"before": 99,
|
||||
"after": 99,
|
||||
"groups_both_by_family": {
|
||||
"access_control": [
|
||||
37,
|
||||
57
|
||||
],
|
||||
"config_secrets": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"deserialization": [
|
||||
32,
|
||||
41
|
||||
],
|
||||
"injection": [
|
||||
53,
|
||||
109
|
||||
],
|
||||
"memory": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"output_encoding": [
|
||||
38,
|
||||
107
|
||||
],
|
||||
"path": [
|
||||
38,
|
||||
109
|
||||
],
|
||||
"unknown": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"untrusted_destination": [
|
||||
42,
|
||||
68
|
||||
]
|
||||
}
|
||||
},
|
||||
"embeddings": {
|
||||
"command": "ousast learn memory embed --budget-usd 0.20 (dry run first: 144,806 estimated tokens, $0.0029)",
|
||||
"requested": 1479,
|
||||
"cached": 1004,
|
||||
"embedded": 475,
|
||||
"calls": 15,
|
||||
"tokens": 150339,
|
||||
"usd": 0.003007
|
||||
},
|
||||
"spend": {
|
||||
"chat_model_calls": 0,
|
||||
"embedding_usd": 0.003007,
|
||||
"github_api": "metadata and raw contents only"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,86 +0,0 @@
|
||||
{
|
||||
"count": 1029,
|
||||
"guard": "green",
|
||||
"sha256": "5a644a1e93c68a0a35c794e653286d0532b7eb8d757123ccce5f08b11bf9a4a2",
|
||||
"sources": {
|
||||
"harvest": {
|
||||
"bytes": 501031,
|
||||
"count": 467,
|
||||
"files": 3,
|
||||
"known_empty": false,
|
||||
"rows": 446,
|
||||
"skipped_values": 446
|
||||
},
|
||||
"measurements": {
|
||||
"bytes": 21169903,
|
||||
"count": 278,
|
||||
"files": 221,
|
||||
"known_empty": false,
|
||||
"rows": 376,
|
||||
"skipped_values": 681
|
||||
},
|
||||
"memory_local": {
|
||||
"bytes": 8084523,
|
||||
"count": 192,
|
||||
"files": 776,
|
||||
"known_empty": false,
|
||||
"rows": 759,
|
||||
"skipped_values": 2
|
||||
},
|
||||
"memory_s3": {
|
||||
"bytes": 15928676,
|
||||
"count": 572,
|
||||
"files": 1499,
|
||||
"known_empty": false,
|
||||
"rows": 1490,
|
||||
"skipped_values": 2
|
||||
},
|
||||
"pairs": {
|
||||
"bytes": 1680480,
|
||||
"count": 728,
|
||||
"files": 28,
|
||||
"known_empty": false,
|
||||
"rows": 900,
|
||||
"skipped_values": 1260
|
||||
},
|
||||
"pointers": {
|
||||
"bytes": 892448,
|
||||
"count": 537,
|
||||
"files": 7,
|
||||
"known_empty": false,
|
||||
"rows": 586,
|
||||
"skipped_values": 666
|
||||
},
|
||||
"populations": {
|
||||
"bytes": 32431,
|
||||
"count": 28,
|
||||
"files": 2,
|
||||
"known_empty": false,
|
||||
"rows": 28,
|
||||
"skipped_values": 0
|
||||
},
|
||||
"sweep": {
|
||||
"bytes": 34006439,
|
||||
"count": 1029,
|
||||
"files": 1619,
|
||||
"known_empty": false,
|
||||
"rows": 2155,
|
||||
"skipped_values": 0
|
||||
},
|
||||
"units": {
|
||||
"bytes": 319641,
|
||||
"count": 229,
|
||||
"files": 9,
|
||||
"known_empty": false,
|
||||
"rows": 642,
|
||||
"skipped_values": 0
|
||||
}
|
||||
},
|
||||
"test_exit_code": 0,
|
||||
"tests": {
|
||||
"errors": 0,
|
||||
"failures": 0,
|
||||
"skipped": 1,
|
||||
"tests": 41
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
{
|
||||
"cleanup": "completed-before-checkpoint",
|
||||
"commit": "5dc86c2f3f54b4dfb59060c88fb3ab5b307ef448",
|
||||
"differences": 0,
|
||||
"image": "ghcr.io/norandom/ousast-engine-task@sha256:b73c9bbd48fbf39c14ac013e5b0ff0a9d536b3d6388b50b1bd4cddfa22264b91",
|
||||
"pins": [
|
||||
{
|
||||
"baseline_digest": "c12c675f816a24ab33a63336a21efdc93db10a37dcf83437d91bec6f728cef1f",
|
||||
"equal": true,
|
||||
"instrument": [
|
||||
{
|
||||
"bytes": 1761,
|
||||
"cpg_bytes": 138723,
|
||||
"jvm": [
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 11.88592994,
|
||||
"success": null
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 35.821929360999995,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 7.4493522819999995,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 2.9803447330000026,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 0.5151375319999971,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 9.691500790999996,
|
||||
"success": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"pin_digest": "63c992b500590a9d4b643c9458df1d6cf71d770b06cc70594acbdba99b06c00b",
|
||||
"record_digest": "c12c675f816a24ab33a63336a21efdc93db10a37dcf83437d91bec6f728cef1f",
|
||||
"seconds": 158.85258329089265
|
||||
},
|
||||
{
|
||||
"baseline_digest": "5a60ed069d99f9325a1d3f3a7612fcf0d853d4fabfd53add81151b051132a5ee",
|
||||
"equal": true,
|
||||
"instrument": [
|
||||
{
|
||||
"bytes": 1791,
|
||||
"cpg_bytes": 138886,
|
||||
"jvm": [
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 7.656527941999999,
|
||||
"success": null
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 27.532885242,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 9.524187851,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 5.0155780199999995,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 0.6809096819999993,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"exit_code": null,
|
||||
"heap_mb": 1700,
|
||||
"seconds": 13.824248304000001,
|
||||
"success": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"pin_digest": "8bd2f48dfb2d00cfc5b60a5dee3a8c87e6970671121caa17221f2449c1254fb2",
|
||||
"record_digest": "5a60ed069d99f9325a1d3f3a7612fcf0d853d4fabfd53add81151b051132a5ee",
|
||||
"seconds": 142.5083134449087
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"commit": "48ae918dec25119142ae781607ba99306ae5ec17",
|
||||
"fixture_bytes": 15374,
|
||||
"fixture_sha256": "dd3c774a47d553105ed85b9cea8810736c233f870e5a015559a1524d5db5a14d",
|
||||
"implementation_sha256": {
|
||||
"benchmarks/ax/batch.py": "37aecac93beb25c02826e81eacaf064fbebb447a998fe139f6d94a07035b8685",
|
||||
"benchmarks/unseen/replay.py": "32efb3e0b9e9d247f2780b5f3ceeb01b4c2cbda3656c75a741f9925621cfdfbc",
|
||||
"benchmarks/unseen/replay_entry.py": "994d8e239eb505dc2c5bad4cdb8b5d45967a8a2aa9ef48fb4c2f20ba45736280",
|
||||
"benchmarks/unseen/score.py": "56e0a7b34482b5d84df39824b8ee447c283e35bd932f38cfe692cf84cb52cef7"
|
||||
},
|
||||
"network_proofs": {
|
||||
"4.4": "coordinator_pending",
|
||||
"5.4": "coordinator_pending"
|
||||
},
|
||||
"scope": "offline synthetic scorer only",
|
||||
"score_command_exit": 0,
|
||||
"scorer_output_sha256": "77f154eedb6a4acd098ab63a0b17434291f0d226dabe8ac66f2e0b362154d002",
|
||||
"scorer_test_exit": 0,
|
||||
"scorer_tests": 14,
|
||||
"synthetic_records": 20,
|
||||
"uncommitted": true,
|
||||
"validation": {
|
||||
"generic_dispatcher_tests": {
|
||||
"exit_code": 0,
|
||||
"passed": 5
|
||||
},
|
||||
"mypy": 0,
|
||||
"proof_and_scorer_tests": {
|
||||
"exit_code": 0,
|
||||
"passed": 16
|
||||
},
|
||||
"requested_pytest": {
|
||||
"exit_code": 0,
|
||||
"passed": 279,
|
||||
"skipped": 8
|
||||
},
|
||||
"reserved_repository_guard": 0,
|
||||
"ruff_check": 0,
|
||||
"ruff_format_check": 0
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
{
|
||||
"biases": [
|
||||
"known_vulnerable_location",
|
||||
"labelled_family",
|
||||
"runnable_project_preference"
|
||||
],
|
||||
"catalogs": {
|
||||
"bytes": 904050,
|
||||
"files": 10,
|
||||
"rows": 873
|
||||
},
|
||||
"eligible_pairs": 192,
|
||||
"eligible_repositories": 192,
|
||||
"exclusions": {
|
||||
"bytes": 0,
|
||||
"count": 0,
|
||||
"files": 0,
|
||||
"ledger_rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"gap_to_target": 0,
|
||||
"guard": {
|
||||
"guard": "green",
|
||||
"test_exit_code": 0,
|
||||
"tests": {
|
||||
"errors": 0,
|
||||
"failures": 0,
|
||||
"skipped": 1,
|
||||
"tests": 41
|
||||
}
|
||||
},
|
||||
"language_floor_gaps": {
|
||||
"java": 0,
|
||||
"javascript": 0,
|
||||
"php": 0,
|
||||
"python": 0,
|
||||
"typescript": 0
|
||||
},
|
||||
"local_metadata": {
|
||||
"config_bytes": 0,
|
||||
"manifest_bytes": 0,
|
||||
"matched_checkouts": 0
|
||||
},
|
||||
"manifest_sha256": "b593d2760f525c38b205fb33866c383fead4666102a84a805150b32c55ceeb62",
|
||||
"per_family": {
|
||||
"injection": 5,
|
||||
"output_encoding": 5,
|
||||
"path": 5,
|
||||
"untrusted_destination": 5
|
||||
},
|
||||
"per_language": {
|
||||
"java": 4,
|
||||
"javascript": 4,
|
||||
"php": 4,
|
||||
"python": 4,
|
||||
"typescript": 4
|
||||
},
|
||||
"purpose": "feasibility_only",
|
||||
"qualification": "not_an_unseen_evaluation_or_block_admission",
|
||||
"rejected": {
|
||||
"commits": 60,
|
||||
"family": 62,
|
||||
"language": 100,
|
||||
"location": 1,
|
||||
"not_development_fix": 453,
|
||||
"repository": 5
|
||||
},
|
||||
"seed": 20261005,
|
||||
"selected": 20,
|
||||
"status": "selected",
|
||||
"target": 20,
|
||||
"unknown_manifest": 20,
|
||||
"unknown_test_suite": 20
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
# Search-with-proof: first `demonstrated` (ceiling proof), 2026-10-06
|
||||
|
||||
First end-to-end `demonstrated` verdict for the search-with-proof verifier. No model, no cluster, $0.
|
||||
Run locally in-process via `openultrasast.search.verify.verify` (non-task bubblewrap lane, userns, node v18.19.1).
|
||||
|
||||
- Pair: glance (jarofghosts/glance), family **path**, index.js `Glance$serveRequest`.
|
||||
- Verdict: **demonstrated** — owned effect in 3/3 affected runs, 0/3 safe runs (zero false proof on the fixed side).
|
||||
- Vulnerable side: canary observed 3/3 (parent traversal reaches the out-of-root nonce file; app reflects it).
|
||||
- Fixed side: 0/3 (the fix commit's parent-escape guard returns 403; the traversal cannot escape).
|
||||
- Wall ~65 s for both sides x 3 runs; ready ~2 s/side.
|
||||
|
||||
## What this proves
|
||||
The execution harness, the path oracle, and the differential verifier are correct and CAN produce a
|
||||
`demonstrated` verdict on a real third-party app. The method's ceiling is reachable. Every prior pilot
|
||||
`inconclusive` on this pair was an instrument artifact, not a method verdict.
|
||||
|
||||
## Two defects that had masked this (both isolated, neither is the method)
|
||||
1. **Broken build.** `[email protected]` depends on an unpinned `mime >=1.2.6`; the checkout's regenerated
|
||||
lockfileVersion-3 pins `[email protected]`, where `.lookup()` was removed. glance crashes serving ANY file
|
||||
(`TypeError: mime.lookup is not a function`). Directory listing (readiness GET /) survives; the first
|
||||
file serve kills the process, so the nonce never reaches the response. Fixed here by installing a
|
||||
compatible `[email protected]` before the run.
|
||||
2. **Wrong demo target.** The stored agent demo traversed to a filename that is not the canary, so it
|
||||
404s even on the vulnerable side. The path oracle is the only owned oracle that hides its target from
|
||||
the demo author. The corrected demo targets the conventional `canary` file one level above the served
|
||||
root (same convention the known-good probe uses).
|
||||
|
||||
## Reproduction
|
||||
Demo schema + driver stored (gitignored) at `benchmarks/search/private/ceiling-proof-2026-10-06/`.
|
||||
Prepared checkouts need their npm symlinks stripped (the verifier refuses links) and a compatible
|
||||
`mime@1`. Payloads stay on disk per the classifier-safety note; never echoed.
|
||||
|
||||
## Follow-on for honest qualification at scale (not covered by this $0 proof)
|
||||
- Keystone: positive-control in `verify.py` so a broken app yields `could_not_run`, never `inconclusive`.
|
||||
- Build-repro: distrust generated npm locks; pin/override incompatible transitive deps; audit all 20 pairs.
|
||||
- Demo-authoring: document the path-oracle target convention so the search AGENT can author a hitting demo.
|
||||
@@ -1,123 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-06 search-with-proof feasibility pilot (path family, vulnerable side, cluster verify lane)",
|
||||
"image": "ghcr.io/norandom/ousast-search-task@sha256:60bfd73f...",
|
||||
"note": "First pilot after the verify-side keystone and the agent-side path-oracle target-parity fix. Feasibility only; payloads/boards live in the gitignored private area.",
|
||||
"ceiling_usd": 3.0,
|
||||
"searches": [
|
||||
{
|
||||
"pair_index": 3,
|
||||
"language": "php",
|
||||
"family": "path",
|
||||
"outcome": "inconclusive",
|
||||
"end_reason": "budget_spent",
|
||||
"end_detail": "demonstrations",
|
||||
"model_calls": 78,
|
||||
"executor_tasks": 12,
|
||||
"verify_tasks": 6,
|
||||
"settled_usd": 0.5779,
|
||||
"wall_seconds": 439,
|
||||
"failure_classes": [
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "no cli entrypoint exists"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify",
|
||||
"class": "noconsistentdifferential"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify",
|
||||
"class": "noconsistentdifferential"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from"
|
||||
},
|
||||
{
|
||||
"phase": "verify",
|
||||
"class": "noconsistentdifferential"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"pair_index": 9,
|
||||
"language": "typescript",
|
||||
"family": "path",
|
||||
"outcome": "plausible_unproven",
|
||||
"end_reason": "budget_spent",
|
||||
"end_detail": "reason_rounds",
|
||||
"model_calls": 154,
|
||||
"executor_tasks": 12,
|
||||
"verify_tasks": 0,
|
||||
"settled_usd": 1.1979,
|
||||
"wall_seconds": 2300,
|
||||
"failure_classes": []
|
||||
},
|
||||
{
|
||||
"pair_index": 13,
|
||||
"language": "python",
|
||||
"family": "path",
|
||||
"outcome": "plausible_unproven",
|
||||
"end_reason": "budget_spent",
|
||||
"end_detail": "run_ceiling",
|
||||
"model_calls": 118,
|
||||
"executor_tasks": 9,
|
||||
"verify_tasks": 0,
|
||||
"settled_usd": 1.1739,
|
||||
"wall_seconds": 564,
|
||||
"failure_classes": [
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "as predicted the parent"
|
||||
},
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "passfalse canary workspacecheckoutcanary passfalse"
|
||||
},
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "with the servedroot prefix"
|
||||
},
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "h check at filepy"
|
||||
},
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "are plain attribute reads"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"aggregate": {
|
||||
"outcomes": {
|
||||
"inconclusive": 1,
|
||||
"plausible_unproven": 2
|
||||
},
|
||||
"demonstrations": 0,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"settled_usd": 2.9496,
|
||||
"searches": 3
|
||||
},
|
||||
"finding": "Parity fix validated: the agent now runs directed, target-aware search toward the named observable (pair 13 reasoned explicitly about reaching it and inspected the app's file-serving guard). No demonstration: all three exhausted their per-pair allowance (demonstrations/reason_rounds/run_ceiling) before a differential; pairs 9 and 13 wrote 0 verify demos (budget consumed in reason/explore). Pair 3 (php) hit entrypoint-discovery failures (no CLI entrypoint / runtime entry missing). Cost ~$0.98/search, higher than the $0.2-0.5 estimate."
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
{
|
||||
"date": "2026-10-06",
|
||||
"image": "ghcr.io/norandom/ousast-search-task@sha256:32c7d23a41936c841b91edf27ded2d1fef5df9d656171e3ae199e7ed51ba0264",
|
||||
"image_tools": {
|
||||
"bwrap": "bubblewrap 0.12.0",
|
||||
"chromium": "Chromium 154.0.8037.92 built on Debian G",
|
||||
"java": "openjdk version \"21.0.12.1\" 2026-08-18\nO",
|
||||
"node": "v20.19.2",
|
||||
"php": "PHP 8.4.26 (cli) (built: Sep 24 2026 17:",
|
||||
"python": "Python 3.13.5",
|
||||
"sqlite": "3.46.1 2024-08-13 09:16:08 c9c2ab54ba1f5",
|
||||
"tsc": "Version 4.9.5"
|
||||
},
|
||||
"isolation_check": {
|
||||
"reason": "task-boundary",
|
||||
"status": "skipped"
|
||||
},
|
||||
"isolation_mode": "task-boundary",
|
||||
"lane": "kube-ax (AX task, gVisor, task-boundary)",
|
||||
"namespaces": {
|
||||
"cgroup": 1,
|
||||
"ipc": 1,
|
||||
"mount": 1,
|
||||
"net": 1,
|
||||
"pid": 1,
|
||||
"setpriv": 127,
|
||||
"user": 1,
|
||||
"uts": 1
|
||||
},
|
||||
"oracles": {
|
||||
"command": {
|
||||
"observer_access": "inconclusive",
|
||||
"real": "demonstrated",
|
||||
"revision_patch": "inconclusive"
|
||||
},
|
||||
"path": {
|
||||
"observer_access": "inconclusive",
|
||||
"real": "demonstrated",
|
||||
"revision_patch": "inconclusive"
|
||||
},
|
||||
"sql": {
|
||||
"observer_access": "inconclusive",
|
||||
"real": "demonstrated",
|
||||
"revision_patch": "inconclusive"
|
||||
},
|
||||
"ssrf": {
|
||||
"observer_access": "inconclusive",
|
||||
"real": "demonstrated",
|
||||
"revision_patch": "inconclusive"
|
||||
},
|
||||
"xss": {
|
||||
"observer_access": "inconclusive",
|
||||
"real": "demonstrated",
|
||||
"revision_patch": "inconclusive"
|
||||
}
|
||||
},
|
||||
"peak_rss_kib": {
|
||||
"children": 460152,
|
||||
"self": 42172,
|
||||
"sum": 502324
|
||||
},
|
||||
"scratch_disk_bytes": 23040,
|
||||
"status": "ok",
|
||||
"task": "search-with-proof 3.2",
|
||||
"wall_seconds": 38.368433234
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-07 FIRST CLUSTER DEMONSTRATION for search-with-proof",
|
||||
"pair": {
|
||||
"index": 13,
|
||||
"repo": "hanxi/xiaomusic",
|
||||
"family": "path",
|
||||
"class": "prefix-check-bypass (startswith without separator)",
|
||||
"language": "python"
|
||||
},
|
||||
"image": "ghcr.io/norandom/ousast-search-task@sha256:e2c8b460...",
|
||||
"side": "vulnerable",
|
||||
"outcome": "demonstrated",
|
||||
"verify_tasks": 18,
|
||||
"model_calls": 55,
|
||||
"settled_usd": 0.3686,
|
||||
"wall_seconds": 2170,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"demo_wires_served_root": true,
|
||||
"finding": "The agent autonomously produced a demonstrated finding on a real CVE fix-pair end to end on the kube-ax cluster: it found the prefix-bypass sink, wired the app's served directory to ${served_root} (prompted by the new served-root control-signal hint after an earlier mis-wire), targeted the oracle's prefix-sibling canary, and the differential verifier confirmed the owned effect on the vulnerable revision and its absence on the fixed revision. Cost $0.37, 55 model calls.",
|
||||
"fix_chain_that_enabled_it": [
|
||||
"parity: agent targets the observable",
|
||||
"verify-time-only: no explore reachability trap",
|
||||
"fair-share budget: authors+verifies within budget",
|
||||
"served-root guidance: wire the served dir to ${served_root}",
|
||||
"oracle prefix-bypass coverage: sibling canary for startswith-without-separator vulns",
|
||||
"control-signal feedback: served-root hint so the agent self-corrects wiring"
|
||||
],
|
||||
"both_sides": {
|
||||
"vulnerable": {
|
||||
"outcome": "demonstrated",
|
||||
"settled_usd": 0.3686,
|
||||
"verify_tasks": 18,
|
||||
"model_calls": 55
|
||||
},
|
||||
"fixed_blind_negative": {
|
||||
"outcome": "inconclusive",
|
||||
"false_proof": false,
|
||||
"settled_usd": 0.5632,
|
||||
"verify_tasks": 18,
|
||||
"model_calls": 85,
|
||||
"note": "Agent searched the FIXED (safe) revision blind (no vuln knowledge); authored demos but the verifier found no effect -> no false proof (Req 6.3 negative)."
|
||||
},
|
||||
"pair_total_settled_usd": 1.224,
|
||||
"verdict": "True positive on the vulnerable revision, zero false proof on the fixed revision."
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-07 search-with-proof validation of the explore-time reachability-trap fix (path family, vulnerable, cluster)",
|
||||
"compares_against": "benchmarks/measurements/2026-10-06-search-pilot-path",
|
||||
"ceiling_usd": 2.0,
|
||||
"searches": [
|
||||
{
|
||||
"pair_index": 9,
|
||||
"language": "typescript",
|
||||
"outcome": "inconclusive",
|
||||
"end_reason": "budget_spent",
|
||||
"end_detail": "reason_rounds",
|
||||
"model_calls": 170,
|
||||
"verify_tasks": 2,
|
||||
"settled_usd": 1.6032,
|
||||
"wall_seconds": 2281,
|
||||
"failure_classes": [
|
||||
{
|
||||
"phase": "explore",
|
||||
"class": "valueerror fact requires tool evidence"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from checkout"
|
||||
},
|
||||
{
|
||||
"phase": "verify/could_not_build",
|
||||
"class": "runtime entry missing from checkout"
|
||||
},
|
||||
{
|
||||
"phase": "verify",
|
||||
"class": "no consistent differential"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"pair_index": 13,
|
||||
"language": "python",
|
||||
"outcome": "plausible_unproven",
|
||||
"end_reason": "budget_spent",
|
||||
"end_detail": "run_ceiling",
|
||||
"model_calls": 36,
|
||||
"verify_tasks": 0,
|
||||
"settled_usd": 0.3418,
|
||||
"wall_seconds": 254,
|
||||
"failure_classes": []
|
||||
}
|
||||
],
|
||||
"aggregate": {
|
||||
"outcomes": {
|
||||
"inconclusive": 1,
|
||||
"plausible_unproven": 1
|
||||
},
|
||||
"demonstrations": 0,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"settled_usd": 1.945
|
||||
},
|
||||
"finding": "Trap fix validated behaviourally. TypeScript pair 9: now AUTHORS a demo (verify_tasks 0->2) where it wrote none before; the demo fails could_not_build ('runtime entry missing from checkout') -- an entrypoint/build issue, same class as the php pair. Python pair 13: no longer burns budget on the reachability dead-end (calls 118->36, $1.17->$0.34); its board shows efficient directed analysis that traced the full route-to-sink path and the app entrypoint, but it hit the per-run budget ceiling (run_ceiling) before authoring/verifying a demo. Still 0 demonstrations, 0 false proofs. Remaining blockers: (1) per-run budget/step allocation leaves no room to author+verify after analysis; (2) demo entrypoint/build (start.path) for compiled/served apps (ts, php)."
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-07 search-with-proof: Python pair re-validation of the per-search fair-share budget fix",
|
||||
"ceiling_usd": 1.5,
|
||||
"pair": {"index": 13, "language": "python", "family": "path"},
|
||||
"outcome": "inconclusive",
|
||||
"end_detail": "demonstrations",
|
||||
"verify_tasks": 18,
|
||||
"model_calls": 98,
|
||||
"settled_usd": 0.7411,
|
||||
"wall_seconds": 2722,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"finding": "Budget fix VALIDATED: with a fair per-search share the Python agent completed the full loop and ran 18 verify tasks (0 before), authoring and verifying demos within budget. Still inconclusive ('no consistent differential'), zero false proofs. Structural root cause confirmed from the authored demo: it does NOT reference ${served_root} anywhere; the agent set the app's served-directory config (an app env var) to a value of its own rather than to the oracle placeholder ${served_root}, so the app served the wrong tree and the planted observable (which lives relative to ${served_root}) was never reachable. Next guidance lever: require the agent to point the application's served/root directory at ${served_root} (via arg, env, or config) for the path oracle, so the observable is in the served tree.",
|
||||
"progression": "parity fix -> agent targets observable; verify-time-only fix -> agent stops the explore reachability trap; budget fair-share -> agent authors+verifies within budget; remaining -> wire served dir to ${served_root}."
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-07 search-with-proof: Python pair re-validation after served-root-wiring guidance fix",
|
||||
"ceiling_usd": 1.5,
|
||||
"pair": {"index": 13, "language": "python", "family": "path"},
|
||||
"outcome": "inconclusive",
|
||||
"end_detail": "run_ceiling",
|
||||
"verify_tasks": 6,
|
||||
"model_calls": 164,
|
||||
"settled_usd": 1.4661,
|
||||
"wall_seconds": 2025,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"demo_wires_served_root": true,
|
||||
"finding": "Served-root fix VALIDATED: the authored demo now references ${served_root} (it did not before). Still inconclusive, 0 false proofs. Two remaining issues: (1) the agent spread its budget across multiple disclosure vectors (the served-root traversal plus a separate temp-path approach) and hit the per-run ceiling (164 calls, $1.47 of $1.50) before converging on one working demo; (2) the committed demo's single traversal step did not produce a differential, so the exact traversal reaching the observable is not yet right. This is now the per-app exploit-authoring + convergence layer, the same layer the $0 local glance ceiling proof handled by a hand-authored demo.",
|
||||
"progression": "parity -> target-aware; verify-time-only -> no explore trap; fair-share budget -> authors+verifies; served-root -> wires the served dir; remaining -> exact traversal + convergence within budget."
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"record": "2026-10-07 xiaomusic vulnerable-side search on the oracle-broadened image (prefix-bypass canary present)",
|
||||
"image": "ghcr.io/norandom/ousast-search-task@sha256:e2c8b460...",
|
||||
"ceiling_usd": 1.5,
|
||||
"outcome": "inconclusive",
|
||||
"verify_tasks": 6,
|
||||
"model_calls": 49,
|
||||
"settled_usd": 0.2922,
|
||||
"false_proofs_fixed_side": 0,
|
||||
"finding": "Oracle broadening landed in the agent: the authored demo now TARGETS the new prefix-sibling vector (references the sibling leak file) instead of the parent-only canary. But it did not demonstrate: the agent set the app's served-directory env var to a value of its OWN instead of the ${served_root} placeholder, so the app served the wrong tree and the sibling canary was outside it (unreachable). This is the served-root-wiring behaviour recurring NONDETERMINISTICALLY (a prior run did wire ${served_root}; this one did not). Cheap ($0.29), ended naturally at 49 calls, not on budget. Remaining gap = agent RELIABILITY on one demo detail: the served-directory config VALUE must be the literal ${served_root}, not the agent's explore-time test path.",
|
||||
"progression": "oracle now covers the vuln class and the agent targets it; last-mile blocker is reliably setting the served-dir config to ${served_root}."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,142 +0,0 @@
|
||||
# Search feasibility tools
|
||||
|
||||
Run these host-side tools from the repository root. Selection and executor smoke
|
||||
make no model calls; the pilot driver calls the configured model unless `--dry-run`
|
||||
is specified.
|
||||
|
||||
## Executor smoke
|
||||
|
||||
```sh
|
||||
python -m benchmarks.search.executor_smoke \
|
||||
--lane ax --image REGISTRY/IMAGE@sha256:DIGEST \
|
||||
--repo-url https://github.com/OWNER/REPOSITORY --commit FULL_SHA \
|
||||
--out /tmp/executor-smoke.json
|
||||
```
|
||||
|
||||
`--lane docker` uses the same object-store mailbox and an already available local
|
||||
image (`--pull never`). Both lanes need the existing batch transport's S3 store
|
||||
configuration; credentials stay on the host. Running this command contacts the
|
||||
store and task runner; the task fetches a shallow HTTPS checkout and installs
|
||||
dependencies. Offline unit tests replace those network operations with a fake lane.
|
||||
|
||||
One managed executor receives clone, list, read, grep, install, trivial run, and
|
||||
stop commands. Acquisition uses `git init` plus a depth-one fetch of the exact
|
||||
commit, retrying initial egress failures for up to 60 seconds. Install recipes are
|
||||
fixed and chosen from root pip/npm/composer/Maven manifests. Caches, temporary
|
||||
files, and pip products live under `/workspace`. A repository without a recognised
|
||||
manifest produces an instrument failure. If no README exists, the smoke reads
|
||||
another listed file and records its size to establish that input was opened.
|
||||
|
||||
The JSON record holds per-command executor and round-trip wall times, exit codes
|
||||
(null for file tools), observed output sizes, and sampled allocated workspace
|
||||
peak, including unlinked command logs. Samples are not an instantaneous disk
|
||||
quota. Output text and presigned URLs are omitted. Zero clone files, a successful
|
||||
install with no output in under one second, command failure, missing runtime
|
||||
output, and cleanup errors produce `instrument_failure` and CLI exit 1. Context
|
||||
cleanup attempts task deletion and removal of mailbox objects on failure as well
|
||||
as success.
|
||||
|
||||
## Pilot selection
|
||||
|
||||
```sh
|
||||
python -m benchmarks.search.pilot_select --seed 20261005
|
||||
```
|
||||
|
||||
Selection uses development catalog metadata only; it performs no downloads or
|
||||
model calls. It excludes unseen reservations and used sets, validates usage
|
||||
ledgers, and runs the existing reserved-repository guard as a black box. Additional
|
||||
local unseen metadata/ledgers can be supplied with `--exclude` and `--ledger`.
|
||||
Use `--cache` to inspect matching local checkouts for manifest/test-suite evidence;
|
||||
the default cache is `~/.cache/openultrasast/repos`. Cached metadata affects
|
||||
preference only and does not establish that the pinned revision builds.
|
||||
The selector never opens population v3 files itself.
|
||||
|
||||
The default identity-bearing manifest is
|
||||
`benchmarks/search/private/pilot.json`, explicitly gitignored. The summary is
|
||||
`benchmarks/measurements/2026-10-05-search-pilot-selection/record.json`; it contains
|
||||
counts, selection biases, input instrumentation, and digests only. Neither output
|
||||
is overwritten. A successful selection is feasibility input, not evidence of
|
||||
buildability, oracle operation, unseen performance, or BLOCK eligibility.
|
||||
|
||||
|
||||
## One-sided pilot driver (task 6.2)
|
||||
|
||||
The shared virtualenv may be installed against another worktree; `PYTHONPATH`
|
||||
below explicitly selects this worktree's source.
|
||||
|
||||
Offline probe, using a scripted model and in-process repository executor with real
|
||||
host verification (no network or provider calls):
|
||||
|
||||
```sh
|
||||
PYTHONPATH="$PWD/src" /home/mc/Source/OpenUltraSAST/.venv/bin/python -m benchmarks.search.pilot_run \
|
||||
--dry-run --side vulnerable --pairs 0 --ceiling-usd 0.50 \
|
||||
--out /tmp/search-pilot-dry.jsonl
|
||||
```
|
||||
|
||||
For three **live** searches, after the oracle infrastructure probe and budget
|
||||
approval, set `SEARCH_TASK_IMAGE` to the tested search image's complete
|
||||
`REGISTRY/IMAGE@sha256:DIGEST` reference, then run:
|
||||
|
||||
```sh
|
||||
PYTHONPATH="$PWD/src" /home/mc/Source/OpenUltraSAST/.venv/bin/python -m benchmarks.search.pilot_run \
|
||||
--manifest benchmarks/search/private/pilot.json \
|
||||
--side vulnerable --pairs 0,1,2 --ceiling-usd 1.50 \
|
||||
--verify-lane vm --image "${SEARCH_TASK_IMAGE:?Set the tested digest-pinned search image}" \
|
||||
--out benchmarks/measurements/2026-10-05-search-pilot-vulnerable.jsonl
|
||||
```
|
||||
|
||||
This command spends money and contacts AX and the store. The ceiling shown is an
|
||||
example for the three $0.50 searches, not evidence that a live run was approved or
|
||||
performed. Use a fresh output path; output files are never overwritten.
|
||||
|
||||
The existing DeepSeek key/base URL environment variables and provider model/prices
|
||||
are reused (`deepseek-flash` by default). `.env` does not override exported values.
|
||||
Retries are disabled. Unknown models require both `--usd-per-mtok-in` and
|
||||
`--usd-per-mtok-out`. Every call reserves its conservative input-token bound plus
|
||||
`--max-tokens` output allowance against the task allowance and shared run ceiling;
|
||||
failed calls and calls with missing usage settle the maximum. Reservations in the
|
||||
record are cumulative admitted maxima, not outstanding spend. Their sum can exceed
|
||||
the ceiling when usage settlements release unused allowances. Settled spend remains
|
||||
bounded by the run ceiling and $0.50/search. Search defaults also cap reason rounds
|
||||
at 4, intents at 3 per round, explore at 15 minutes including acquisition, and demos
|
||||
at 3. Physical AX submissions and coordinator steps have separate 32-task caps.
|
||||
|
||||
The board defaults to a private local FileStore under a fresh directory in
|
||||
`benchmarks/search/private/runs/`; `--board-memory` selects another board store URI.
|
||||
The AX mailbox still requires the existing `OUSAST_MEMORY=s3://bucket/prefix`, public
|
||||
HTTPS `S3_ENDPOINT`, and host store credentials. Board storage does not replace the
|
||||
mailbox. Executors receive only presigned object URLs. The brain runs on the VM;
|
||||
repository tools and build/export run in key-free AX executors. VM verification
|
||||
requires working user namespaces and consumes executor-built archives. `--verify-lane
|
||||
ax` switches to the existing dispatcher, with three fresh verifier tasks per side
|
||||
and no executor holding a worker slot during those repetitions.
|
||||
|
||||
Each selected index produces a JSON line followed by one run summary: Req 3.4
|
||||
outcome, reserved/settled spend, calls, tokens, physical tasks, coordinator steps,
|
||||
phase timings, failure codes, input bytes, board bytes, end reason, and hashed
|
||||
repository/revision identities. The private board and checkouts contain identity-bearing
|
||||
data. The public JSONL omits raw model/executor diagnostics. Selection advantages
|
||||
are explicit in the summary; this is feasibility evidence, not BLOCK qualification.
|
||||
|
||||
A fixed invocation creates a fresh board/demo and retains only the fixed revision.
|
||||
Its verifier runs two independent sets of repetitions on that same fixed revision;
|
||||
no vulnerable checkout or previously generated vulnerable demo is passed in.
|
||||
`fixed_effect_observed` reports a consistent owned effect on the first three fixed
|
||||
runs (`null` when unavailable or inconsistent). Such an effect requires adjudication;
|
||||
it is not promoted to a differential `demonstrated` outcome. The normal safe probe
|
||||
therefore ends `inconclusive`, never “not vulnerable”.
|
||||
|
||||
Generic `injection` needs an `oracle` value of `sql` or `command` in the private
|
||||
manifest; without a subtype verification returns `no_oracle`. No subtype is guessed
|
||||
from repository names. The selected manifest's pair 1 currently lacks this metadata.
|
||||
Live transport, provider authentication/cost, real project buildability, and AX
|
||||
oracle operation must still be measured; an offline probe cannot establish them.
|
||||
|
||||
## Status (2026-10-07)
|
||||
|
||||
The feasibility pilot achieved its first cluster `demonstrated` finding (xiaomusic, path /
|
||||
prefix-check-bypass): vulnerable side demonstrated, fixed side blind search produced no
|
||||
false proof. See `benchmarks/measurements/2026-10-07-search-first-cluster-demonstration/`.
|
||||
Outcomes and spend are aggregates only; stored demos and boards stay in the gitignored
|
||||
`benchmarks/search/private/` area. Pilot exit (≥5 demonstrations / 0 false proofs per
|
||||
family) and the ≥73-negative BLOCK qualification remain open.
|
||||
@@ -1,241 +0,0 @@
|
||||
"""Replay a stored pilot demo without a board or model; output DIR/record.json.
|
||||
|
||||
Live execution acquires repositories inside executor tasks. Offline tests inject
|
||||
an in-process executor and a fake lane; this CLI itself is not an offline clone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import DockerLane
|
||||
from benchmarks.search.pilot_run import (
|
||||
FAMILY_ORACLES,
|
||||
AXLane,
|
||||
AXSideDispatcher,
|
||||
BrowserExecutor,
|
||||
Pilot,
|
||||
Side,
|
||||
Workload,
|
||||
diagnostic,
|
||||
digest,
|
||||
exception_reason,
|
||||
load_demo,
|
||||
validate_image,
|
||||
validate_oracle,
|
||||
validate_result,
|
||||
verification_run_seconds,
|
||||
verification_timeout,
|
||||
verify,
|
||||
)
|
||||
from openultrasast.search.verify import SideRecord
|
||||
|
||||
|
||||
class Replay(Pilot):
|
||||
"""Reuse pilot acquisition/redaction without constructing its board or worker."""
|
||||
|
||||
def __init__(self, args, root, lane, demo):
|
||||
self.args, self.root, self.lane = args, root, lane
|
||||
self.deadline = float("inf")
|
||||
self.row = dict(repo=args.repo_url, vulnerable=args.commit, fixed=args.fixed_commit or args.commit, file=demo["start"]["path"])
|
||||
self.record = dict(tasks_submitted=0, executor_tasks=0, verify_tasks=0, input_bytes=0, failures=[])
|
||||
self.preparations = []
|
||||
|
||||
def prepare(self, revision, spec):
|
||||
row = dict(
|
||||
revision_digest=digest(revision),
|
||||
wall_seconds_by_phase={},
|
||||
build_exit_code=None,
|
||||
build_stderr_tail="",
|
||||
archive_bytes=0,
|
||||
archive_parts=0,
|
||||
)
|
||||
self.preparations.append(row)
|
||||
started = time.monotonic()
|
||||
task = None
|
||||
try:
|
||||
with self._executor(revision) as task:
|
||||
row["wall_seconds_by_phase"]["acquisition"] = time.monotonic() - started
|
||||
before = time.monotonic()
|
||||
try:
|
||||
return task.prepare(spec)
|
||||
finally:
|
||||
row["wall_seconds_by_phase"]["prepare_roundtrip"] = time.monotonic() - before
|
||||
except Exception as exc:
|
||||
self.failure("build_export", exc)
|
||||
# The dispatcher preserves build failure semantics and we retain diagnostics.
|
||||
raise ValueError(self.record["failures"][-1]["reason"]) from exc
|
||||
finally:
|
||||
result = getattr(task, "preparation_result", {})
|
||||
row["wall_seconds_by_phase"].update(result.get("wall_seconds_by_phase", {}))
|
||||
row.update(
|
||||
build_exit_code=result.get("exit_code"),
|
||||
build_stderr_tail=result.get("stderr", ""),
|
||||
archive_bytes=result.get("archive_bytes", 0),
|
||||
archive_parts=result.get("archive_parts", 0),
|
||||
wall_seconds=time.monotonic() - started,
|
||||
)
|
||||
|
||||
|
||||
def run_replay(args, root, *, lane=None):
|
||||
started = time.monotonic()
|
||||
demo = load_demo(args.demo)
|
||||
timeout = verification_timeout(demo)
|
||||
replay = Replay(args, root, lane, demo)
|
||||
record = dict(
|
||||
type="demo_replay",
|
||||
repo_digest=digest(args.repo_url),
|
||||
revision_digest=digest(args.commit),
|
||||
family=args.family,
|
||||
lane=args.lane,
|
||||
both=args.both,
|
||||
model_calls=0,
|
||||
spend_usd=0,
|
||||
)
|
||||
if args.both:
|
||||
record["fixed_revision_digest"] = digest(args.fixed_commit)
|
||||
if not FAMILY_ORACLES[args.family]:
|
||||
result = SideRecord("no_oracle", (), 0, "no configured owned oracle")
|
||||
sides = [] if args.both else [result]
|
||||
else:
|
||||
validate_oracle(args.family, demo["oracle"])
|
||||
workload = Workload(
|
||||
args.image,
|
||||
frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}),
|
||||
verification_run_seconds(demo, timeout) + 60,
|
||||
validate_result,
|
||||
kind="search-verify",
|
||||
)
|
||||
lane = lane or (DockerLane if args.lane == "docker" else AXLane)(args, workload)
|
||||
if isinstance(lane, DockerLane):
|
||||
lane._diagnostics = []
|
||||
lane._prepare_image()
|
||||
lane._prepared = True
|
||||
replay.lane = copy.copy(lane)
|
||||
if isinstance(lane, DockerLane):
|
||||
replay.lane.ax = replay.lane._docker_command
|
||||
keys = [Side(root / "affected"), Side(root / "fixed")]
|
||||
pins = {keys[0].checkout: args.commit, keys[1].checkout: args.fixed_commit}
|
||||
|
||||
def counted_lane(*a, **kw):
|
||||
replay.admit_task()
|
||||
replay.record["verify_tasks"] += 1
|
||||
return lane(*a, **kw)
|
||||
|
||||
dispatcher = AXSideDispatcher(
|
||||
args, args.image, root / "verify", lane=counted_lane, prepare=lambda side, spec: replay.prepare(pins[side.checkout], spec)
|
||||
)
|
||||
|
||||
def dispatch(**kwargs):
|
||||
before = time.monotonic()
|
||||
try:
|
||||
return dispatcher(**kwargs)
|
||||
except Exception as exc:
|
||||
replay.failure("verify", exc)
|
||||
return SideRecord(
|
||||
"could_not_run", (), time.monotonic() - before, replay.record["failures"][-1]["reason"], isolation_mode="task-boundary"
|
||||
)
|
||||
|
||||
if args.both:
|
||||
result = verify(
|
||||
*keys,
|
||||
args.demo,
|
||||
args.family,
|
||||
timeout_seconds=timeout,
|
||||
oracle=demo["oracle"],
|
||||
task_dispatcher=dispatch,
|
||||
browser=BrowserExecutor(task_boundary=True) if demo["oracle"] == "xss" else None,
|
||||
)
|
||||
sides = result.sides
|
||||
else:
|
||||
result = dispatch(side=keys[0], demo=demo, family=args.family, timeout_seconds=timeout, fresh_task=True)
|
||||
sides = [result]
|
||||
phases = {}
|
||||
for preparation in replay.preparations:
|
||||
for phase, seconds in preparation["wall_seconds_by_phase"].items():
|
||||
phases[phase] = phases.get(phase, 0) + seconds
|
||||
for phase in ("build", "ready", "run"):
|
||||
phases["verify_" + phase] = sum(getattr(s, phase + "_seconds") for s in sides)
|
||||
record.update(
|
||||
outcome=result.outcome,
|
||||
reason=result.reason,
|
||||
sides=[asdict(s) for s in sides],
|
||||
preparations=replay.preparations,
|
||||
**replay.record,
|
||||
wall_seconds_by_phase=phases,
|
||||
wall_seconds=time.monotonic() - started,
|
||||
)
|
||||
# Apply the pilot's diagnostic redaction to every free-form string, including evidence.
|
||||
private = tuple(str(v) for v in replay.row.values()) + (str(root), args.repo_url.removeprefix("https://github.com/"))
|
||||
|
||||
def redact(value):
|
||||
if isinstance(value, str):
|
||||
return diagnostic(value, maximum=1500, private=private)
|
||||
if isinstance(value, list):
|
||||
return [redact(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: redact(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
return redact(record)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for flag in ("repo-url", "commit", "image"):
|
||||
parser.add_argument("--" + flag, required=True)
|
||||
parser.add_argument("--demo", type=Path, required=True)
|
||||
parser.add_argument("--family", choices=tuple(FAMILY_ORACLES), required=True)
|
||||
parser.add_argument("--lane", choices=("ax", "docker"), required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--both", action="store_true")
|
||||
parser.add_argument("--fixed-commit")
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--atespace", default="default")
|
||||
parser.add_argument("--kubeconfig")
|
||||
parser.add_argument("--docker-image-bytes", type=int)
|
||||
args = parser.parse_args(argv)
|
||||
if args.both != bool(args.fixed_commit):
|
||||
parser.error("--both and --fixed-commit must be supplied together")
|
||||
if not re.fullmatch(r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", args.repo_url):
|
||||
parser.error("credential-free GitHub HTTPS repository URL required")
|
||||
if any(not re.fullmatch(r"[0-9a-f]{40}", pin) for pin in (args.commit, args.fixed_commit) if pin is not None):
|
||||
parser.error("full lowercase commit SHA required")
|
||||
try:
|
||||
validate_image(args.image)
|
||||
load_demo(args.demo)
|
||||
except (ValueError, OSError) as exc:
|
||||
parser.error(str(exc))
|
||||
if args.lane == "docker" and (args.docker_image_bytes is None or args.docker_image_bytes <= 0):
|
||||
parser.error("docker requires --docker-image-bytes (unpacked image upper bound)")
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="demo-replay-") as directory:
|
||||
try:
|
||||
record = run_replay(args, Path(directory))
|
||||
except Exception as exc:
|
||||
record = dict(
|
||||
type="demo_replay",
|
||||
repo_digest=digest(args.repo_url),
|
||||
revision_digest=digest(args.commit),
|
||||
outcome="could_not_run",
|
||||
reason=exception_reason(exc, private=(args.repo_url, args.commit, args.fixed_commit or "", directory)),
|
||||
model_calls=0,
|
||||
spend_usd=0,
|
||||
sides=[],
|
||||
)
|
||||
if args.both:
|
||||
record["fixed_revision_digest"] = digest(args.fixed_commit)
|
||||
(args.out / "record.json").write_text(json.dumps(record, indent=2, allow_nan=False) + "\n")
|
||||
print(json.dumps(record, allow_nan=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,190 +0,0 @@
|
||||
"""One no-model executor task; run as python -m benchmarks.search.executor_smoke.
|
||||
|
||||
Runtime acquisition/install needs network inside the executor. Unit tests are offline.
|
||||
Records contain measurements and sanitized failure tails, never presigned bearer URLs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import tarfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from benchmarks.ax.batch import AXLane, DockerLane, SearchExecutorTask, Workload, validate_image
|
||||
from openultrasast.search.task_storage import diagnostic, exception_reason
|
||||
|
||||
# URL and pin are positional shell arguments, never interpolated into shell code.
|
||||
# Retry the initial acquisition for delayed task egress activation (at most 60 s).
|
||||
CLONE = """git init -q .
|
||||
git remote add origin "$1"
|
||||
end=$(($(date +%s) + 60))
|
||||
until git -c protocol.version=2 fetch --depth=1 origin "$2"; do
|
||||
test "$(date +%s)" -lt "$end" || exit 1
|
||||
sleep 3
|
||||
done
|
||||
git checkout -q --detach FETCH_HEAD
|
||||
test "$(git rev-parse HEAD)" = "$2"
|
||||
"""
|
||||
|
||||
|
||||
def install_recipe(files):
|
||||
"""Fixed recipes, selected from checkout root manifests only."""
|
||||
files = set(files)
|
||||
if "requirements.txt" in files:
|
||||
return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "-r", "requirements.txt"]
|
||||
if files & {"pyproject.toml", "setup.py", "setup.cfg"}:
|
||||
return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "."]
|
||||
if "package.json" in files:
|
||||
return "npm", ["npm", "ci" if "package-lock.json" in files else "install"]
|
||||
if "composer.json" in files:
|
||||
return "composer", ["composer", "install", "--no-interaction", "--prefer-dist"]
|
||||
if "pom.xml" in files:
|
||||
return "maven", ["mvn", "-B", "-Dmaven.repo.local=/workspace/.m2", "-DskipTests", "package"]
|
||||
raise ValueError("no recognised install manifest")
|
||||
|
||||
|
||||
def run_smoke(lane, repo_url, commit, *, deadline=900):
|
||||
record = {
|
||||
"status": "instrument_failure",
|
||||
"commands": [],
|
||||
"scratch_peak_bytes": 0,
|
||||
"scratch_measurement": "sampled workspace allocated bytes plus command logs",
|
||||
"model_calls": 0,
|
||||
}
|
||||
parts = urlsplit(repo_url)
|
||||
private = tuple(p for p in parts.path.split("/") if p) + (parts.path.rsplit("/", 1)[-1].removesuffix(".git"),)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
parts = urlsplit(repo_url)
|
||||
if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment:
|
||||
raise ValueError("credential-free HTTPS repository URL required")
|
||||
if not re.fullmatch("[0-9a-f]{40}", commit):
|
||||
raise ValueError("full lowercase commit SHA required")
|
||||
archive = io.BytesIO()
|
||||
with tarfile.open(fileobj=archive, mode="w"):
|
||||
pass
|
||||
# The empty checkout is filled by the first command inside the key-free task.
|
||||
with SearchExecutorTask(lane, archive.getvalue(), deadline=deadline) as task:
|
||||
record["task"] = task.name
|
||||
|
||||
def command(step, name, args):
|
||||
remaining = deadline - (time.monotonic() - started) - 2
|
||||
if remaining <= 0:
|
||||
raise TimeoutError("smoke deadline")
|
||||
timeout = min(300, remaining)
|
||||
if name == "run":
|
||||
args = {**args, "timeout_seconds": timeout}
|
||||
before = time.monotonic()
|
||||
try:
|
||||
result = task.client.submit(
|
||||
name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3}
|
||||
)
|
||||
except Exception as exc:
|
||||
result = {
|
||||
"error": type(exc).__name__,
|
||||
"reason": exception_reason(exc, private=private),
|
||||
"phase": "timeout" if isinstance(exc, TimeoutError) else "result upload",
|
||||
"exit_code": None,
|
||||
"stderr": "",
|
||||
}
|
||||
output_bytes = sum(result.get(k + "_bytes", len(result.get(k, "").encode())) for k in ("stdout", "stderr"))
|
||||
if name != "run":
|
||||
output_bytes = len(json.dumps(result).encode())
|
||||
row = {
|
||||
"step": step,
|
||||
"command": name,
|
||||
"wall_seconds": result.get("wall_seconds", time.monotonic() - before),
|
||||
"roundtrip_seconds": time.monotonic() - before,
|
||||
"exit_code": result.get("exit_code"),
|
||||
"output_bytes": output_bytes,
|
||||
"scratch_peak_bytes": result.get("scratch_peak_bytes"),
|
||||
"truncated": result.get("truncated", False),
|
||||
}
|
||||
if "input_bytes" in result:
|
||||
row["input_bytes"] = result["input_bytes"]
|
||||
failed = (
|
||||
result.get("error")
|
||||
or result.get("timed_out")
|
||||
or result.get("status") == "could_not_build"
|
||||
or (name == "run" and result.get("exit_code") != 0)
|
||||
)
|
||||
if failed:
|
||||
row.update(
|
||||
phase=result.get("phase", "timeout" if result.get("timed_out") else "run"),
|
||||
reason=diagnostic(
|
||||
result.get("reason") or result.get("error") or f"CommandFailure: exit code {result.get('exit_code')}",
|
||||
private=private,
|
||||
),
|
||||
stderr=diagnostic(result.get("stderr", ""), maximum=4000, private=private),
|
||||
)
|
||||
record["commands"].append(row)
|
||||
record["scratch_peak_bytes"] = max(record["scratch_peak_bytes"], result.get("scratch_peak_bytes", 0))
|
||||
if result.get("error") or result.get("timed_out") or result.get("status") == "could_not_build":
|
||||
raise ValueError(step + " executor failure: " + row["reason"])
|
||||
if name == "run" and result.get("exit_code") != 0:
|
||||
raise ValueError(step + " nonzero or missing exit code: " + row["reason"])
|
||||
if step == "install" and output_bytes == 0 and row["wall_seconds"] < 1:
|
||||
raise ValueError("empty fast install: exit 0 with empty output in under 1 s")
|
||||
return result
|
||||
|
||||
command("clone", "run", {"command": ["sh", "-ec", CLONE, "clone", repo_url, commit]})
|
||||
files = command("list_files", "list_files", {}).get("files", [])
|
||||
if not files:
|
||||
raise ValueError("clone yielded zero files")
|
||||
record["files_listed"] = len(files)
|
||||
readme = next((p for p in files if Path(p).name.lower().startswith("readme")), None)
|
||||
# Always open input even when the repository has no README.
|
||||
read = command("read_file", "read_file", {"path": readme or files[0]})
|
||||
record["readme_present"] = readme is not None
|
||||
record["input_bytes"] = read.get("input_bytes")
|
||||
if not isinstance(record["input_bytes"], int):
|
||||
raise ValueError("file read lacks input size")
|
||||
command("grep", "grep", {"pattern": "test"})
|
||||
recipe, argv = install_recipe(files)
|
||||
record["recipe"] = recipe
|
||||
command("install", "run", {"command": argv})
|
||||
trivial = ["node", "-e", "console.log(1)"] if recipe == "npm" else ["python3", "-c", "print(1)"]
|
||||
ran = command("trivial", "run", {"command": trivial})
|
||||
if ran.get("stdout", "").strip() != "1":
|
||||
raise ValueError("trivial command did not print 1")
|
||||
if not command("stop", "stop", {}).get("stopped"):
|
||||
raise ValueError("stop not acknowledged")
|
||||
record["status"] = "ok"
|
||||
except Exception as exc:
|
||||
record["status"] = "instrument_failure"
|
||||
record["reason"] = exception_reason(exc, private=private)
|
||||
record["wall_seconds"] = time.monotonic() - started
|
||||
return record
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--lane", choices=("ax", "docker"), required=True)
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--repo-url", required=True)
|
||||
parser.add_argument("--commit", required=True)
|
||||
parser.add_argument("--out", type=Path, required=True, help="One JSON record")
|
||||
parser.add_argument("--deadline", type=float, default=900)
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--atespace", default="default")
|
||||
parser.add_argument("--kubeconfig")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
validate_image(args.image)
|
||||
lane = (DockerLane if args.lane == "docker" else AXLane)(args, Workload(args.image, frozenset(), args.deadline, None))
|
||||
record = run_smoke(lane, args.repo_url, args.commit, deadline=args.deadline)
|
||||
except Exception as exc:
|
||||
record = {"status": "instrument_failure", "reason": "lane setup: " + exception_reason(exc)}
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps(record))
|
||||
return int(record["status"] != "ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,573 +0,0 @@
|
||||
"""One-sided feasibility searches. Live mode is opt-in by omitting --dry-run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import tarfile
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import AXLane, ClusterUnavailable, SearchExecutorTask, Workload, validate_image
|
||||
from benchmarks.ax.search_verify import AXSideDispatcher, validate_result
|
||||
from benchmarks.search.executor_smoke import CLONE
|
||||
from openultrasast.config import load_dotenv
|
||||
from openultrasast.model.endpoint import (
|
||||
DEEPSEEK_BASE_ENV,
|
||||
DEEPSEEK_BASE_URL,
|
||||
DEEPSEEK_KEY_ENV,
|
||||
DEFAULT_DETECTOR_MODEL,
|
||||
Prices,
|
||||
price_of,
|
||||
)
|
||||
from openultrasast.plane.budget import BudgetExhausted, SpendBudget
|
||||
from openultrasast.plane.memory import FileStore, open_store
|
||||
from openultrasast.provider.openrouter import OpenRouterChatClient
|
||||
from openultrasast.search import _sandbox
|
||||
from openultrasast.search.board import Board
|
||||
from openultrasast.search.budget import SearchBudget, verification_run_seconds, verification_timeout
|
||||
from openultrasast.search.coordinator import Coordinator
|
||||
from openultrasast.search.demo import FAMILY_ORACLES, load_demo, validate_oracle
|
||||
from openultrasast.search.executor import InProcessExecutor
|
||||
from openultrasast.search.oracles import BrowserExecutor
|
||||
from openultrasast.search.probe import materialise
|
||||
from openultrasast.search.task_storage import diagnostic, exception_reason
|
||||
from openultrasast.search.verify import Side, VerificationRecord, verify
|
||||
from openultrasast.search.verify_task import extract_checkout, validate_spec
|
||||
from openultrasast.search.worker import StubModel, Worker
|
||||
|
||||
|
||||
def digest(value):
|
||||
return hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
|
||||
def tool(name, **args):
|
||||
return {"tool_calls": [{"id": "call", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}]}
|
||||
|
||||
|
||||
def scripted(schema):
|
||||
return StubModel(
|
||||
[
|
||||
tool("read_file", path="app.py"),
|
||||
tool("write_demo", schema=schema),
|
||||
tool("finish", fact={"text": "Candidate demo", "evidence_refs": ["tool:1"], "demo": "demo/"}),
|
||||
*[{"content": "{}"} for _ in range(4)],
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class RepositoryFailure(ValueError):
|
||||
"""The selected checkout or candidate cannot be read."""
|
||||
|
||||
|
||||
class Pilot:
|
||||
def __init__(self, args, row, index, root, store, run_budget, client, prices, lane=None):
|
||||
self.args, self.root, self.lane = args, root, lane
|
||||
# Fixed searches retain no vulnerable pin, even in trusted verification state.
|
||||
self.row = {k: v for k, v in row.items() if args.side != "fixed" or k != "vulnerable"}
|
||||
self.phase = "setup"
|
||||
self.deadline = float("inf")
|
||||
self.record = dict(
|
||||
type="search",
|
||||
pair_index=index,
|
||||
side=args.side,
|
||||
repo_digest=digest(row["repo"]),
|
||||
revision_digest=digest(row[args.side]),
|
||||
family=row["family"],
|
||||
language=row["language"],
|
||||
dry_run=args.dry_run,
|
||||
fixed_effect_observed=None,
|
||||
outcome="no_evidence",
|
||||
end_reason="exhausted",
|
||||
end_detail=None,
|
||||
task_metrics=[],
|
||||
executor_task_metrics=[],
|
||||
tasks_submitted=0,
|
||||
executor_tasks=0,
|
||||
verify_tasks=0,
|
||||
input_bytes=0,
|
||||
wall_seconds_by_phase={},
|
||||
failures=[],
|
||||
)
|
||||
self.board = Board(store, "pilot-" + uuid.uuid4().hex, coordinator="pilot")
|
||||
self.board.commit(
|
||||
writer="pilot",
|
||||
hints=[
|
||||
dict(
|
||||
id="candidate",
|
||||
step="reason",
|
||||
task_id="manifest",
|
||||
text=json.dumps({k: row[k] for k in ("family", "file", "function", "oracle") if k in row}),
|
||||
)
|
||||
],
|
||||
)
|
||||
self.worker = Worker(
|
||||
client,
|
||||
repo=root / "unmounted",
|
||||
demo=root / "demo",
|
||||
model=args.model,
|
||||
prices=prices,
|
||||
run_budget=run_budget,
|
||||
max_output_tokens=args.max_tokens,
|
||||
)
|
||||
self.family = row["family"]
|
||||
self.sides = None
|
||||
self.verifications = []
|
||||
|
||||
def failure(self, phase, reason, command=None):
|
||||
private = tuple(str(v) for k, v in self.row.items() if k in {"repo", "file", "function", "vulnerable", "fixed"})
|
||||
private += (str(self.root), self.row["repo"].removeprefix("https://github.com/"))
|
||||
if isinstance(reason, BaseException):
|
||||
safe = exception_reason(reason, private=private)
|
||||
elif isinstance(reason, str) and re.match(r"^[A-Za-z_]\w*: ", reason):
|
||||
# Worker errors already carry a type. Preserve it when identity redaction
|
||||
# expands the message beyond the public record's tail limit.
|
||||
kind, message = reason.split(": ", 1)
|
||||
safe = kind + ": " + diagnostic(message, maximum=max(0, 298 - len(kind)), private=private)
|
||||
else:
|
||||
safe = diagnostic(reason, private=private)
|
||||
entry = dict(phase=phase, reason=safe)
|
||||
command = command if command is not None else reason
|
||||
if hasattr(command, "exit_code"):
|
||||
entry.update(
|
||||
command_phase=diagnostic(command.phase),
|
||||
exit_code=command.exit_code,
|
||||
stderr=diagnostic(command.stderr, maximum=1500, private=private),
|
||||
stdout=diagnostic(getattr(command, "stdout", ""), maximum=1500, private=private),
|
||||
ready_seconds=getattr(command, "ready_seconds", 0),
|
||||
)
|
||||
if (phase == "verify" or phase.startswith("verify/")) and entry.get("stderr"):
|
||||
entry["reason"] = entry["stderr"][-300:]
|
||||
self.record["failures"].append(entry)
|
||||
|
||||
@contextmanager
|
||||
def executor(self, revision, deadline=900):
|
||||
before = time.monotonic()
|
||||
metrics = dict(self.worker.metrics)
|
||||
self.executor_end = "ok"
|
||||
submitted = self.record["executor_tasks"]
|
||||
try:
|
||||
with self._executor(revision, deadline) as task:
|
||||
yield task
|
||||
except BudgetExhausted as exc:
|
||||
self.executor_end = exc.end_detail
|
||||
raise
|
||||
except Exception:
|
||||
self.executor_end = "execution_failure"
|
||||
raise
|
||||
finally:
|
||||
if self.record["executor_tasks"] > submitted:
|
||||
self.record["executor_task_metrics"].append(
|
||||
{
|
||||
"executor_task": submitted + 1,
|
||||
**{key: value - metrics[key] for key, value in self.worker.metrics.items()},
|
||||
"wall_seconds": time.monotonic() - before,
|
||||
"end": self.executor_end,
|
||||
}
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _executor(self, revision, deadline=900):
|
||||
deadline = min(deadline, self.deadline - time.monotonic())
|
||||
if deadline < 1:
|
||||
raise BudgetExhausted("task deadline", usd=0, calls=0, end_detail="task_wall")
|
||||
acquired_at = time.monotonic()
|
||||
empty = io.BytesIO()
|
||||
with tarfile.open(fileobj=empty, mode="w"):
|
||||
pass
|
||||
self.admit_task()
|
||||
self.record["executor_tasks"] += 1
|
||||
with SearchExecutorTask(self.lane, empty.getvalue(), deadline=deadline) as task:
|
||||
remaining = deadline - (time.monotonic() - acquired_at)
|
||||
if remaining < 1:
|
||||
raise BudgetExhausted("acquisition deadline", usd=0, calls=0, end_detail="task_wall")
|
||||
timeout = min(180, remaining)
|
||||
result = task.client.submit(
|
||||
"run",
|
||||
{"command": ["sh", "-ec", CLONE, "clone", self.row["repo"], revision], "timeout_seconds": timeout},
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
if result.get("exit_code") != 0 or result.get("error"):
|
||||
self.failure("acquisition", "checkout_command_failed")
|
||||
raise RepositoryFailure("checkout failed")
|
||||
remaining = deadline - (time.monotonic() - acquired_at)
|
||||
if remaining < 1:
|
||||
raise BudgetExhausted("acquisition deadline", usd=0, calls=0, end_detail="task_wall")
|
||||
result = task.client.submit("read_file", {"path": self.row["file"]}, timeout_seconds=min(30, remaining))
|
||||
if not isinstance(result.get("input_bytes"), int) or result["input_bytes"] <= 0:
|
||||
self.failure("acquisition", "candidate_input_unreadable")
|
||||
raise RepositoryFailure("unreadable input")
|
||||
self.record["input_bytes"] += result["input_bytes"]
|
||||
yield task
|
||||
|
||||
def admit_task(self):
|
||||
if self.record["tasks_submitted"] >= 32:
|
||||
raise BudgetExhausted("executor task cap", usd=0, calls=0, end_detail="tasks")
|
||||
self.record["tasks_submitted"] += 1
|
||||
|
||||
def prepare(self, revision, spec):
|
||||
before = time.monotonic()
|
||||
try:
|
||||
with self.executor(revision) as task:
|
||||
return task.prepare(spec)
|
||||
except BudgetExhausted:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.failure("build_export", exc)
|
||||
raise
|
||||
finally:
|
||||
wall = self.record["wall_seconds_by_phase"]
|
||||
wall["build_export"] = wall.get("build_export", 0) + time.monotonic() - before
|
||||
|
||||
def verification(self, limits=None):
|
||||
if not FAMILY_ORACLES[self.family]:
|
||||
return VerificationRecord("no_oracle", (), 0, "no configured owned oracle")
|
||||
demo = load_demo(self.root / "demo")
|
||||
validate_oracle(self.family, demo["oracle"])
|
||||
timeout = verification_timeout(demo, limits or self.worker.limits)
|
||||
if self.args.dry_run:
|
||||
sides = self.sides if self.args.side == "vulnerable" else (self.sides[1], self.sides[1])
|
||||
return verify(*sides, self.root / "demo", self.family, timeout_seconds=timeout, oracle=demo["oracle"])
|
||||
revisions = [self.row[self.args.side], self.row["fixed"]]
|
||||
if self.args.verify_lane == "ax":
|
||||
# Side paths are opaque lookup keys; no checkout or opposite revision reaches the brain.
|
||||
sides = [Side(self.root / str(i)) for i in range(2)]
|
||||
pins = {s.checkout: pin for s, pin in zip(sides, revisions, strict=True)}
|
||||
parent = self
|
||||
verify_lane = copy.copy(self.lane)
|
||||
verify_lane.workload = Workload(
|
||||
self.args.image,
|
||||
frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}),
|
||||
verification_run_seconds(demo, timeout) + 60,
|
||||
validate_result,
|
||||
kind="search-verify",
|
||||
)
|
||||
|
||||
class CountingLane:
|
||||
def __call__(self, *a, **kw):
|
||||
parent.admit_task()
|
||||
parent.record["verify_tasks"] += 1
|
||||
return verify_lane(*a, **kw)
|
||||
|
||||
dispatcher = AXSideDispatcher(
|
||||
self.args,
|
||||
self.args.image,
|
||||
self.root / "verify",
|
||||
lane=CountingLane(),
|
||||
prepare=lambda side, spec: self.prepare(pins[side.checkout], spec),
|
||||
)
|
||||
return verify(
|
||||
*sides,
|
||||
self.root / "demo",
|
||||
self.family,
|
||||
timeout_seconds=timeout,
|
||||
oracle=demo["oracle"],
|
||||
task_dispatcher=dispatcher,
|
||||
browser=BrowserExecutor(task_boundary=True) if demo["oracle"] == "xss" else None,
|
||||
)
|
||||
if _sandbox.isolation_mode() != "userns":
|
||||
raise _sandbox.IsolationUnavailable("VM verification requires userns")
|
||||
sides = []
|
||||
spec = dict(demo=demo, family=self.family, timeout_seconds=timeout)
|
||||
for revision in revisions:
|
||||
archive = self.prepare(revision, spec)
|
||||
bundle = self.root / ("built-" + uuid.uuid4().hex)
|
||||
extract_checkout(archive, bundle)
|
||||
sides.append(Side(bundle / "checkout", products=bundle / "products"))
|
||||
# Export contains the executor's build-free schema.
|
||||
(self.root / "verify-demo").mkdir(exist_ok=True)
|
||||
built_spec = validate_spec(json.loads((bundle / "spec.json").read_text()))
|
||||
expected_demo = {**demo, "build": {"recipe": "none", "arguments": []}}
|
||||
if built_spec != {**spec, "demo": expected_demo}:
|
||||
raise ValueError("executor altered verification contract")
|
||||
(self.root / "verify-demo/demo.json").write_text(json.dumps(built_spec["demo"]))
|
||||
return verify(
|
||||
*sides,
|
||||
self.root / "verify-demo",
|
||||
self.family,
|
||||
timeout_seconds=timeout,
|
||||
oracle=demo["oracle"],
|
||||
browser=BrowserExecutor() if demo["oracle"] == "xss" else None,
|
||||
)
|
||||
|
||||
def dispatch(self, task):
|
||||
self.phase = task.step
|
||||
before = time.monotonic()
|
||||
self.deadline = before + task.limits.get("task_wall_seconds", 900)
|
||||
metrics_before = dict(self.worker.metrics)
|
||||
result = {}
|
||||
end = "execution_failure"
|
||||
try:
|
||||
if task.step == "verify":
|
||||
result = self.verification(task.limits)
|
||||
self.verifications.append(result.outcome)
|
||||
self.record["outcome"] = result.outcome
|
||||
if self.args.side == "fixed" and result.sides:
|
||||
observations = result.sides[0]
|
||||
if observations.outcome == "observed" and len(observations.runs) == 3:
|
||||
effects = {r.observed for r in observations.runs}
|
||||
if len(effects) == 1:
|
||||
self.record["fixed_effect_observed"] = effects.pop()
|
||||
wall = self.record["wall_seconds_by_phase"]
|
||||
for phase in ("build", "ready", "run"):
|
||||
wall["verify_" + phase] = wall.get("verify_" + phase, 0) + sum(getattr(s, phase + "_seconds") for s in result.sides)
|
||||
|
||||
for side in result.sides:
|
||||
if side.outcome != "observed":
|
||||
self.failure("verify/" + side.outcome, side.reason, command=side)
|
||||
if result.outcome != "demonstrated":
|
||||
self.failure("verify", "no_consistent_differential" if result.outcome == "inconclusive" else result.outcome)
|
||||
end = result.outcome
|
||||
return dict(
|
||||
status="ok",
|
||||
outcome=result.outcome,
|
||||
evidence_refs=["verify:owned"],
|
||||
cost_usd=0,
|
||||
control_observed=result.sides[0].control_observed if result.sides else None,
|
||||
)
|
||||
if task.step == "explore":
|
||||
if self.args.dry_run:
|
||||
self.worker.executor = InProcessExecutor(self.sides[0 if self.args.side == "vulnerable" else 1].checkout)
|
||||
result = self.worker(task)
|
||||
else:
|
||||
with self.executor(self.row[self.args.side], deadline=task.limits["task_wall_seconds"]) as remote:
|
||||
self.worker.executor = remote.client
|
||||
remaining = task.limits["task_wall_seconds"] - (time.monotonic() - before)
|
||||
if remaining < 1:
|
||||
raise BudgetExhausted("explore deadline", usd=0, calls=0, end_detail="task_wall")
|
||||
result = self.worker(replace(task, limits={**task.limits, "task_wall_seconds": remaining}))
|
||||
self.executor_end = result.get("end_detail") or result.get("status", "execution_failure")
|
||||
else:
|
||||
result = self.worker(task)
|
||||
end = result.get("end_detail") or result.get("status", "execution_failure")
|
||||
if result.get("status") != "ok":
|
||||
self.failure(task.step, result.get("failure", "worker_execution_failed"))
|
||||
if self.worker.metrics["model_calls"] == 0 and not result.get("budget_exhaustion"):
|
||||
raise RuntimeError(self.record["failures"][-1]["reason"])
|
||||
if not self.verifications:
|
||||
self.record["outcome"] = "inconclusive"
|
||||
return result
|
||||
except BudgetExhausted as exc:
|
||||
end = exc.end_detail
|
||||
raise
|
||||
finally:
|
||||
self.record["task_metrics"].append(
|
||||
{
|
||||
"task_id": task.id,
|
||||
"step": task.step,
|
||||
**{key: value - metrics_before[key] for key, value in self.worker.metrics.items()},
|
||||
"wall_seconds": time.monotonic() - before,
|
||||
"end": end,
|
||||
}
|
||||
)
|
||||
wall = self.record["wall_seconds_by_phase"]
|
||||
wall[task.step] = wall.get(task.step, 0) + time.monotonic() - before
|
||||
|
||||
def run(self):
|
||||
started = time.monotonic()
|
||||
coordinator = Coordinator(self.board, self.dispatch, budget=SearchBudget(retries=0))
|
||||
try:
|
||||
state = coordinator.run()
|
||||
self.record["end_reason"] = state["end_reason"]
|
||||
self.record["end_detail"] = state.get("end_detail")
|
||||
if not self.verifications and any(f.get("step") == "explore" for f in state["facts"]):
|
||||
self.record["outcome"] = "plausible_unproven"
|
||||
except Exception as exc:
|
||||
self.failure(self.phase, exc)
|
||||
self.record["outcome"] = (
|
||||
"could_not_build"
|
||||
if self.record["failures"] and any(f["phase"] in ("build_export", "acquisition") for f in self.record["failures"])
|
||||
else "inconclusive"
|
||||
)
|
||||
self.record["end_reason"] = "instrument_failure"
|
||||
if isinstance(exc, ClusterUnavailable) or (self.worker.metrics["model_calls"] == 0 and not isinstance(exc, RepositoryFailure)):
|
||||
self.record["aborted_reason"] = self.record["failures"][-1]["reason"]
|
||||
metrics = self.worker.metrics
|
||||
self.record.update(
|
||||
spend={k: metrics[k] for k in ("reserved", "settled")},
|
||||
**{k: metrics[k] for k in ("model_calls", "tokens_in", "tokens_out")},
|
||||
coordinator_tasks=coordinator.progress["tasks"],
|
||||
board_bytes=len(json.dumps(self.board.state).encode()),
|
||||
wall_seconds=time.monotonic() - started,
|
||||
)
|
||||
return self.record
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--side", choices=("vulnerable", "fixed"), required=True)
|
||||
parser.add_argument("--pairs", required=True)
|
||||
parser.add_argument("--ceiling-usd", required=True, type=float)
|
||||
parser.add_argument("--manifest", type=Path, default=Path("benchmarks/search/private/pilot.json"))
|
||||
parser.add_argument("--private-root", type=Path, default=Path("benchmarks/search/private/runs"))
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--board-memory", help="Memory store URI; defaults to a private local FileStore")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--verify-lane", choices=("vm", "ax"), default="vm")
|
||||
parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
|
||||
parser.add_argument("--max-tokens", type=int, default=4096)
|
||||
parser.add_argument("--usd-per-mtok-in", type=float)
|
||||
parser.add_argument("--usd-per-mtok-out", type=float)
|
||||
parser.add_argument("--image")
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--atespace", default="default")
|
||||
parser.add_argument("--kubeconfig")
|
||||
args = parser.parse_args(argv)
|
||||
if not math.isfinite(args.ceiling_usd) or args.ceiling_usd < 0:
|
||||
parser.error("ceiling must be finite and nonnegative")
|
||||
remaining_ceiling = args.ceiling_usd
|
||||
if not re.fullmatch(r"\d+(,\d+)*", args.pairs) or args.max_tokens < 1:
|
||||
parser.error("pairs must be comma-separated nonnegative indices; max tokens must be positive")
|
||||
indices = [int(i) for i in args.pairs.split(",")]
|
||||
if len(indices) != len(set(indices)):
|
||||
parser.error("duplicate pair indices")
|
||||
prices = price_of(args.model)
|
||||
if args.usd_per_mtok_in is not None or args.usd_per_mtok_out is not None:
|
||||
if any(v is None or not math.isfinite(v) or v < 0 for v in (args.usd_per_mtok_in, args.usd_per_mtok_out)):
|
||||
parser.error("both finite nonnegative prices required")
|
||||
prices = Prices(args.usd_per_mtok_in, args.usd_per_mtok_in, args.usd_per_mtok_out)
|
||||
if prices is None:
|
||||
parser.error("unknown model: supply both price flags")
|
||||
lane = None
|
||||
if args.dry_run:
|
||||
if indices != [0] or args.verify_lane != "vm" or args.board_memory:
|
||||
parser.error("dry run uses probe index 0, VM verifier and private FileStore only")
|
||||
rows = [
|
||||
dict(
|
||||
repo="probe",
|
||||
vulnerable="probe-before",
|
||||
fixed="probe-after",
|
||||
family="path",
|
||||
language="python",
|
||||
file="app.py",
|
||||
function="main",
|
||||
)
|
||||
]
|
||||
else:
|
||||
try:
|
||||
validate_image(args.image or "")
|
||||
except ValueError:
|
||||
parser.error("image must be pinned by sha256 digest")
|
||||
if not args.private_root.resolve().is_relative_to(Path("benchmarks/search/private").resolve()):
|
||||
parser.error("live working files must stay in benchmarks/search/private")
|
||||
if args.verify_lane == "vm":
|
||||
if _sandbox.isolation_mode() != "userns":
|
||||
parser.error("VM verification requires userns")
|
||||
try:
|
||||
_sandbox.isolation_check(timeout_seconds=5)
|
||||
except _sandbox.IsolationUnavailable:
|
||||
parser.error("VM userns isolation probe failed")
|
||||
# Validate private manifest before opening any remote store or client.
|
||||
if not args.manifest.resolve().is_relative_to(Path("benchmarks/search/private").resolve()):
|
||||
parser.error("manifest must stay in benchmarks/search/private")
|
||||
try:
|
||||
rows = json.loads(args.manifest.read_bytes())["pairs"]
|
||||
if not isinstance(rows, list) or any(index >= len(rows) for index in indices):
|
||||
parser.error("pair index outside manifest")
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
parser.error("private manifest unreadable or invalid")
|
||||
for index in indices:
|
||||
row = rows[index]
|
||||
if not isinstance(row, dict) or not all(
|
||||
isinstance(row.get(k), str) and row[k] for k in ("repo", args.side, "fixed", "family", "language", "file", "function")
|
||||
):
|
||||
parser.error("invalid manifest pair")
|
||||
if row["family"] not in (
|
||||
"injection",
|
||||
"path",
|
||||
"output_encoding",
|
||||
"untrusted_destination",
|
||||
"sql",
|
||||
"command",
|
||||
"xss",
|
||||
"ssrf",
|
||||
"access_control",
|
||||
"deserialization",
|
||||
"configuration",
|
||||
"secrets",
|
||||
):
|
||||
parser.error("unknown manifest family")
|
||||
if row["language"] not in ("python", "javascript", "typescript", "php", "java"):
|
||||
parser.error("unknown manifest language")
|
||||
path = Path(row["file"])
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in row["file"]:
|
||||
parser.error("invalid candidate path")
|
||||
|
||||
if not re.fullmatch(r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", row["repo"]):
|
||||
parser.error("invalid repository URL")
|
||||
if not all(re.fullmatch("[0-9a-f]{40}", row[s]) for s in (args.side, "fixed")):
|
||||
parser.error("invalid revision")
|
||||
load_dotenv()
|
||||
if not os.environ.get(DEEPSEEK_KEY_ENV):
|
||||
parser.error("DeepSeek key is not configured")
|
||||
client = OpenRouterChatClient(
|
||||
api_key=os.environ[DEEPSEEK_KEY_ENV], base_url=os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), max_attempts=1
|
||||
)
|
||||
root = args.private_root / uuid.uuid4().hex
|
||||
root.mkdir(parents=True, mode=0o700)
|
||||
records = []
|
||||
started = time.monotonic()
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.out.open("x") as output:
|
||||
aborted_reason = None
|
||||
try:
|
||||
store = open_store(args.board_memory) if args.board_memory else FileStore(root / "boards")
|
||||
if not args.dry_run:
|
||||
lane = AXLane(
|
||||
args, Workload(args.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 900, validate_result, kind="search-verify")
|
||||
)
|
||||
except Exception as exc:
|
||||
aborted_reason = exception_reason(exc)
|
||||
for position, index in enumerate(indices if aborted_reason is None else []):
|
||||
local = root / str(index)
|
||||
local.mkdir()
|
||||
if args.dry_run:
|
||||
sides, demos = materialise(local / "probe", "path")
|
||||
client = scripted(load_demo(demos["real"]))
|
||||
searches_remaining = len(indices) - position
|
||||
per_search = remaining_ceiling / searches_remaining if searches_remaining > 0 and remaining_ceiling > 0 else 0.0
|
||||
pilot = Pilot(args, rows[index], index, local, store, SpendBudget(per_search), client, prices, lane)
|
||||
if args.dry_run:
|
||||
pilot.sides = sides
|
||||
pilot.record["input_bytes"] = len((sides[0 if args.side == "vulnerable" else 1].checkout / "app.py").read_bytes())
|
||||
record = pilot.run()
|
||||
remaining_ceiling = max(0.0, remaining_ceiling - record["spend"]["settled"])
|
||||
records.append(record)
|
||||
output.write(json.dumps(record, allow_nan=False) + "\n")
|
||||
output.flush()
|
||||
if record.get("aborted_reason"):
|
||||
aborted_reason = record["aborted_reason"]
|
||||
break
|
||||
summary = dict(
|
||||
type="summary",
|
||||
end_reason="instrument_failure" if aborted_reason else "completed",
|
||||
aborted_reason=aborted_reason,
|
||||
skipped_pairs=indices[len(records) :],
|
||||
searches=len(records),
|
||||
dry_run=args.dry_run,
|
||||
side=args.side,
|
||||
ceiling_usd=args.ceiling_usd,
|
||||
purpose="feasibility_only",
|
||||
advantages=["known_location", "labelled_family", "runnable_project_selection_bias"],
|
||||
fixed_effects_observed=sum(r["fixed_effect_observed"] is True for r in records),
|
||||
outcomes=dict(Counter(r["outcome"] for r in records)),
|
||||
spend={k: sum(r["spend"][k] for r in records) for k in ("reserved", "settled")},
|
||||
**{k: sum(r[k] for r in records) for k in ("model_calls", "tokens_in", "tokens_out", "tasks_submitted")},
|
||||
wall_seconds=time.monotonic() - started,
|
||||
)
|
||||
output.write(json.dumps(summary, allow_nan=False) + "\n")
|
||||
return int(bool(aborted_reason) or any(r["end_reason"] == "instrument_failure" for r in records))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,370 +0,0 @@
|
||||
"""Offline development-pair feasibility pilot; identities stay in a gitignored manifest.
|
||||
|
||||
Run from the repository root with python -m benchmarks.search.pilot_select.
|
||||
The existing reserved guard is executed as a black box: this module never reads v3.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from benchmarks.unseen import eligibility, ledger
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
from openultrasast.search.task_storage import exception_reason
|
||||
|
||||
LANGUAGES = ("python", "javascript", "typescript", "php", "java")
|
||||
FAMILIES = ("injection", "path", "output_encoding", "untrusted_destination")
|
||||
SHA = re.compile(r"[0-9a-f]{40}")
|
||||
MANIFESTS = {
|
||||
"requirements.txt": "pip",
|
||||
"pyproject.toml": "pip",
|
||||
"setup.py": "pip",
|
||||
"Pipfile": "pip",
|
||||
"package.json": "npm",
|
||||
"composer.json": "composer",
|
||||
"pom.xml": "maven",
|
||||
"build.gradle": "gradle",
|
||||
"build.gradle.kts": "gradle",
|
||||
}
|
||||
|
||||
|
||||
def encoded(value) -> bytes:
|
||||
return (json.dumps(value, sort_keys=True, indent=2) + "\n").encode()
|
||||
|
||||
|
||||
def safe(path: Path) -> bool:
|
||||
return not eligibility.excluded(path) and not eligibility.excluded(path.resolve()) and not path.is_symlink()
|
||||
|
||||
|
||||
def load_catalogs(root: Path) -> tuple[list[dict], dict]:
|
||||
"""Read raw metadata only; PairCase materialization would unnecessarily open source excerpts."""
|
||||
rows = []
|
||||
counts = {"files": 0, "bytes": 0, "rows": 0}
|
||||
directory = (root / "benchmarks/pairs").resolve()
|
||||
for path in sorted(directory.rglob("catalog.toml")):
|
||||
if not safe(path) or not path.resolve().is_relative_to(directory):
|
||||
continue
|
||||
raw = path.read_bytes()
|
||||
items = eligibility.parse(path, raw).get("pair", [])
|
||||
if not isinstance(items, list) or not all(isinstance(row, dict) for row in items):
|
||||
raise ValueError("invalid_catalog")
|
||||
rows.extend(items)
|
||||
counts["files"] += 1
|
||||
counts["bytes"] += len(raw)
|
||||
counts["rows"] += len(items)
|
||||
if not counts["bytes"] or not rows:
|
||||
raise ValueError("empty_catalogs")
|
||||
return rows, counts
|
||||
|
||||
|
||||
def exclusions(root: Path, extra: tuple[Path, ...] = (), ledgers: tuple[Path, ...] = ()) -> tuple[set[str], dict]:
|
||||
"""Unseen-local reservations/used sets, including private journals; never the development used set.
|
||||
|
||||
eligibility.build_used collects development data itself, so its union cannot exclude a
|
||||
development pilot. The unseen usage ledger has opaque slices, not identities: every pool
|
||||
is excluded in full regardless of usage, and ledger integrity is checked separately.
|
||||
"""
|
||||
directory = root / "benchmarks/unseen"
|
||||
paths = {
|
||||
p
|
||||
for p in directory.rglob("*")
|
||||
if p.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"} and "fixtures" not in p.relative_to(directory).parts
|
||||
}
|
||||
paths.update(extra)
|
||||
ledger_paths = set(ledgers)
|
||||
ledger_paths.update(p for p in paths if any(word in p.stem.lower() for word in ("ledger", "usage")))
|
||||
paths.update(ledger_paths)
|
||||
names: set[str] = set()
|
||||
report = {"files": 0, "bytes": 0, "ledger_rows": 0}
|
||||
for path in sorted(paths):
|
||||
if not safe(path):
|
||||
raise ValueError("unsafe_exclusion_source")
|
||||
raw = path.read_bytes()
|
||||
report["files"] += 1
|
||||
report["bytes"] += len(raw)
|
||||
if path in ledger_paths:
|
||||
report["ledger_rows"] += len(ledger.read(path))
|
||||
else:
|
||||
value = eligibility.parse(path, raw)
|
||||
|
||||
# TOML [[repository]] is a collection, whereas repositories() treats
|
||||
# a scalar "repository" field as an identity. Walk containers too.
|
||||
def collect(item):
|
||||
names.update(eligibility.repositories(item))
|
||||
if isinstance(item, str) and eligibility.NAME.fullmatch(item):
|
||||
names.add(eligibility.normalize(item))
|
||||
if isinstance(item, dict):
|
||||
for child in item.values():
|
||||
if isinstance(child, (dict, list)):
|
||||
collect(child)
|
||||
elif isinstance(item, list):
|
||||
for child in item:
|
||||
collect(child)
|
||||
|
||||
collect(value)
|
||||
report.update(count=len(names), sha256=hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest())
|
||||
return names, report
|
||||
|
||||
|
||||
def cache_safe(path: Path) -> bool:
|
||||
# Git config is intentionally readable here; every other exclusion and all
|
||||
# symlink ancestors remain forbidden before opening files or listing trees.
|
||||
lexical = path.absolute()
|
||||
resolved = path.resolve()
|
||||
return not any("v3" in part.lower() or part in {"__pycache__", ".venv"} for p in (lexical, resolved) for part in p.parts) and not any(
|
||||
p.is_symlink() for p in (lexical, *lexical.parents)
|
||||
)
|
||||
|
||||
|
||||
def enrich(rows: list[dict], caches: tuple[Path, ...], *, excluded: set[str] | None = None) -> dict:
|
||||
"""Prefer metadata actually present in a matching local clone, without fetching.
|
||||
|
||||
This is only a preference: a cached checkout is not proof the pinned revision builds.
|
||||
Only development-candidate paths are opened; unrelated cached repositories are not read.
|
||||
"""
|
||||
report = {"matched_checkouts": 0, "config_bytes": 0, "manifest_bytes": 0}
|
||||
seen = {}
|
||||
for row in rows:
|
||||
try:
|
||||
name = eligibility.normalize(row.get("repo", ""))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if name in (excluded or set()):
|
||||
continue
|
||||
if name not in seen:
|
||||
seen[name] = []
|
||||
for cache in caches:
|
||||
for checkout in (cache / name, cache / name.rsplit("/", 1)[-1]):
|
||||
config = checkout / ".git/config"
|
||||
if not cache_safe(config) or not config.is_file():
|
||||
continue
|
||||
parser = configparser.ConfigParser()
|
||||
raw = config.read_bytes()
|
||||
parser.read_string(raw.decode())
|
||||
origin = parser.get('remote "origin"', "url", fallback="")
|
||||
try:
|
||||
matching = eligibility.normalize(origin) == name
|
||||
except ValueError:
|
||||
matching = False
|
||||
if not matching:
|
||||
continue
|
||||
files = []
|
||||
for manifest in MANIFESTS:
|
||||
path = checkout / manifest
|
||||
if cache_safe(path) and path.is_file():
|
||||
report["manifest_bytes"] += len(path.read_bytes())
|
||||
files.append(manifest)
|
||||
for folder in ("tests", "test", "spec", "__tests__", "src/test"):
|
||||
directory = checkout / folder
|
||||
if cache_safe(directory) and directory.is_dir() and any(directory.iterdir()):
|
||||
files.append(folder + "/")
|
||||
seen[name] = files
|
||||
report["matched_checkouts"] += 1
|
||||
report["config_bytes"] += len(raw)
|
||||
break
|
||||
if seen[name]:
|
||||
break
|
||||
row["files"] = sorted(set(row.get("files", []) + seen[name]))
|
||||
return report
|
||||
|
||||
|
||||
def candidates(rows: list[dict], excluded: set[str]) -> tuple[list[dict], dict]:
|
||||
taxonomy = load_families()
|
||||
used = eligibility.UsedSet(sources={"unseen": excluded})
|
||||
out, rejected = [], Counter()
|
||||
for row in rows:
|
||||
reason = None
|
||||
try:
|
||||
raw_repo = row.get("repo", "")
|
||||
repo = eligibility.normalize(raw_repo)
|
||||
if "://" in raw_repo and not raw_repo.startswith("https://"):
|
||||
raise ValueError
|
||||
url = raw_repo if raw_repo.startswith("https://") else "https://github.com/" + repo
|
||||
if repo.count("/") != 1:
|
||||
raise ValueError
|
||||
parsed = urlsplit(url)
|
||||
if (
|
||||
parsed.netloc != "github.com"
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or parsed.path.strip("/").removesuffix(".git").lower() != repo
|
||||
):
|
||||
raise ValueError
|
||||
if row.get("fix_repo") and eligibility.normalize(row["fix_repo"]) != repo:
|
||||
raise ValueError
|
||||
except (ValueError, TypeError):
|
||||
rejected["repository"] += 1
|
||||
continue
|
||||
if row.get("split", "train") != "train" or row.get("provenance") == "synthetic":
|
||||
reason = "not_development_fix"
|
||||
elif row.get("language") not in LANGUAGES:
|
||||
reason = "language"
|
||||
elif eligibility.rejection(eligibility.Resolution(repo, frozenset({repo}), repo), used, set()):
|
||||
reason = "unseen"
|
||||
vulnerable, fixed = row.get("parent", ""), row.get("commit", "")
|
||||
if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (vulnerable, fixed)) or vulnerable == fixed:
|
||||
reason = reason or "commits"
|
||||
expected = row.get("expected", [])
|
||||
families = {row["family"]} if row.get("family") else set()
|
||||
for item in [row, *expected]:
|
||||
family = taxonomy.family_of_cwe(item.get("cwe", ""))
|
||||
if family:
|
||||
families.add(family.id)
|
||||
families &= set(FAMILIES)
|
||||
function = row.get("function") or next((r.get("function") for r in expected if r.get("function")), "")
|
||||
file = row.get("path") or row.get("relpath", "")
|
||||
if len(families) != 1:
|
||||
reason = reason or "family"
|
||||
if not function or function in {"<anon>", "<module>", "*"} or not file:
|
||||
reason = reason or "location"
|
||||
if reason:
|
||||
rejected[reason] += 1
|
||||
continue
|
||||
files = row.get("files", [])
|
||||
manifest = next((MANIFESTS[Path(f).name] for f in sorted(files) if Path(f).name in MANIFESTS), None)
|
||||
tests = True if any(re.search(r"(^|/)(tests?|__tests__|spec)(/|$)|(^|/)test_[^/]+", f) for f in files) else None
|
||||
out.append(
|
||||
{
|
||||
"repo": url.removesuffix(".git").rstrip("/"),
|
||||
"vulnerable": vulnerable,
|
||||
"fixed": fixed,
|
||||
"family": next(iter(families)),
|
||||
"language": row["language"],
|
||||
"file": file,
|
||||
"function": function,
|
||||
"manifest": manifest,
|
||||
"has_tests": tests,
|
||||
}
|
||||
)
|
||||
return out, dict(sorted(rejected.items()))
|
||||
|
||||
|
||||
def select(rows: list[dict], *, seed: int, count: int = 20) -> list[dict]:
|
||||
"""Balance language/family representation, then prefer evidenced runnable projects.
|
||||
|
||||
Seeded SHA ordering is invariant to catalog ordering and Python hash randomization.
|
||||
At most one pair per normalized repository is selected.
|
||||
"""
|
||||
if count < 1:
|
||||
raise ValueError("invalid_count")
|
||||
remaining = sorted(rows, key=lambda row: hashlib.sha256(str(seed).encode() + encoded(row)).digest())
|
||||
selected, used = [], set()
|
||||
languages, families = Counter(), Counter()
|
||||
while remaining and len(selected) < count:
|
||||
row = min(
|
||||
remaining,
|
||||
key=lambda r: (
|
||||
languages[r["language"]] >= 3,
|
||||
languages[r["language"]],
|
||||
families[r["family"]],
|
||||
not (r["has_tests"] and r["manifest"]),
|
||||
not r["manifest"],
|
||||
not r["has_tests"],
|
||||
),
|
||||
)
|
||||
selected.append(row)
|
||||
used.add(eligibility.normalize(row["repo"]))
|
||||
languages[row["language"]] += 1
|
||||
families[row["family"]] += 1
|
||||
remaining = [r for r in remaining if eligibility.normalize(r["repo"]) not in used]
|
||||
return selected
|
||||
|
||||
|
||||
def summary(rows: list[dict], *, seed: int, manifest: bytes, target: int = 20) -> dict:
|
||||
languages = Counter(row["language"] for row in rows)
|
||||
return {
|
||||
"status": "selected" if len(rows) == target else "insufficient_candidates",
|
||||
"purpose": "feasibility_only",
|
||||
"seed": seed,
|
||||
"target": target,
|
||||
"selected": len(rows),
|
||||
"gap_to_target": max(0, target - len(rows)),
|
||||
"per_language": {lang: languages[lang] for lang in LANGUAGES},
|
||||
"per_family": {family: sum(r["family"] == family for r in rows) for family in FAMILIES},
|
||||
"language_floor_gaps": {lang: max(0, 3 - languages[lang]) for lang in LANGUAGES},
|
||||
"unknown_test_suite": sum(r["has_tests"] is None for r in rows),
|
||||
"unknown_manifest": sum(r["manifest"] is None for r in rows),
|
||||
"manifest_sha256": hashlib.sha256(manifest).hexdigest(),
|
||||
"biases": ["known_vulnerable_location", "labelled_family", "runnable_project_preference"],
|
||||
"qualification": "not_an_unseen_evaluation_or_block_admission",
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--seed", type=int, default=20261005)
|
||||
parser.add_argument("--count", type=int, default=20)
|
||||
parser.add_argument("--exclude", type=Path, action="append", default=[])
|
||||
parser.add_argument("--ledger", type=Path, action="append", default=[])
|
||||
parser.add_argument("--cache", type=Path, action="append", default=[])
|
||||
parser.add_argument("--manifest", type=Path, default=Path("benchmarks/search/private/pilot.json"))
|
||||
parser.add_argument("--output", type=Path, default=Path("benchmarks/measurements/2026-10-05-search-pilot-selection/record.json"))
|
||||
args = parser.parse_args(argv)
|
||||
private = args.root / args.manifest
|
||||
created = False
|
||||
try:
|
||||
if not private.resolve().is_relative_to((args.root / "benchmarks/search/private").resolve()):
|
||||
raise ValueError("manifest_must_be_private")
|
||||
ignored = subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=args.root, capture_output=True)
|
||||
if ignored.returncode != 0:
|
||||
raise ValueError("manifest_not_gitignored")
|
||||
guard = eligibility.verify_tests(args.root)
|
||||
rows, instrument = load_catalogs(args.root)
|
||||
excluded, exclusion_report = exclusions(args.root, tuple(args.exclude), tuple(args.ledger))
|
||||
# The black-box guard has already approved the development catalogs.
|
||||
metadata = enrich(rows, tuple(args.cache) or (Path.home() / ".cache/openultrasast/repos",), excluded=excluded)
|
||||
available, rejected = candidates(rows, excluded)
|
||||
selected = select(available, seed=args.seed, count=args.count)
|
||||
payload = encoded({"seed": args.seed, "pairs": selected})
|
||||
private.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with private.open("xb") as handle:
|
||||
created = True
|
||||
os.chmod(private, 0o600)
|
||||
handle.write(payload)
|
||||
# The URL-bearing private selection now lies in the guard's development tree,
|
||||
# so even catalog bare-name aliases are checked without reading v3 ourselves.
|
||||
guard = eligibility.verify_tests(args.root)
|
||||
record = summary(selected, seed=args.seed, manifest=payload, target=args.count)
|
||||
record.update(
|
||||
catalogs=instrument,
|
||||
exclusions=exclusion_report,
|
||||
rejected=rejected,
|
||||
local_metadata=metadata,
|
||||
eligible_pairs=len(available),
|
||||
eligible_repositories=len({r["repo"] for r in available}),
|
||||
guard=guard,
|
||||
)
|
||||
output = args.root / args.output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output.open("xb") as handle:
|
||||
handle.write(encoded(record))
|
||||
print(encoded(record).decode(), end="")
|
||||
return 0 if len(selected) == args.count else 2
|
||||
except Exception as exc:
|
||||
if created:
|
||||
private.unlink(missing_ok=True)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "instrument_failure",
|
||||
"reason": exception_reason(
|
||||
exc, private=tuple(part for row in locals().get("rows", []) for part in str(row.get("repo", "")).split("/") if part)
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,375 +0,0 @@
|
||||
# Unseen evaluation, group 1
|
||||
|
||||
The coordinator runs task 1.5 from the repository root:
|
||||
|
||||
```sh
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH=$PWD/src
|
||||
python -m benchmarks.unseen.eligibility --s3-store s3:// --verify-tests --output benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json
|
||||
```
|
||||
|
||||
This is the networked command; implementation tests use fakes. It loads `.env` with
|
||||
`openultrasast.config.load_dotenv` (exported values win), opens the local FileStore and
|
||||
S3 MemoryStore through `openultrasast.plane.memory.open_store(..., read_only=True)`,
|
||||
and uses `GH_TOKEN` for GitHub GET resolution. `s3://` selects `S3_BUCKET`; pass
|
||||
`--s3-store s3://BUCKET/PREFIX` for a prefixed store and `--local-store file:///PATH`
|
||||
for a different local store. Read-only S3 opening skips the normal write/Select probe;
|
||||
collection uses LIST/GET only. Existing normal store opening is unchanged.
|
||||
|
||||
Pair catalogs/recipes include local pointer rows (`vendored = false`, including
|
||||
`fix_repo`). Additional external manifests can be supplied with repeatable
|
||||
`--pointer-manifest PATH`. Registered units in memory and experiment YAML are followed;
|
||||
missing units with a registered digest fail. Plans without frozen units are not used data.
|
||||
Experiment units identify repositories in `group`. Binary measurement artifacts are
|
||||
read and counted; structured JSON, TOML and multi-document YAML are parsed strictly.
|
||||
|
||||
Each source records files/objects read, bytes read, repository-bearing rows (distinct
|
||||
names per input), and distinct repository counts. The digest is SHA-256 of sorted,
|
||||
normalized names joined with newlines. Source counts overlap. An empty source fails;
|
||||
only after establishing it is empty may the coordinator pass `--known-empty LABEL`.
|
||||
The output records that declaration. Missing/unreadable inputs still fail.
|
||||
|
||||
GitHub redirects add the requested and canonical names, plus parent/source fork
|
||||
identities. Used names are also resolved so an old used name excludes a renamed
|
||||
candidate. GitHub cannot enumerate all historical names: injected clients may supply
|
||||
`former_names`; actual historical aliases come from used references and redirects.
|
||||
A 404 for a historical/synthetic used identity retains its literal identity; other
|
||||
API failures stop collection. Candidate resolution always fails closed on a 404.
|
||||
`resolve(candidate, api)` and `rejection(resolution, used, selected_networks)` are
|
||||
available to the later sourcing task. Names stay in memory; output contains only
|
||||
counts, labels and SHA-256 digests. Output files are created exclusively, never overwritten.
|
||||
|
||||
`--verify-tests` runs the three group-1 test modules with both output streams discarded,
|
||||
including the existing population guard. Only aggregate test counts and the exit code
|
||||
enter the record; a failed test prevents the record. V3 paths are excluded before
|
||||
eligibility reads; only the existing guard handles v3 inside its black-box test process.
|
||||
|
||||
Population reservations retain their existing sweep over `benchmarks/`, `src/`, and
|
||||
`tests/`. Pool reservations also cover `plane/`. Each reservation exempts only its own
|
||||
directory; population checks include all of `benchmarks/unseen/private/`. This preserves
|
||||
the existing population behavior while adding pools (the hard rule for this increment).
|
||||
|
||||
All `pool-pN.toml` files require a matching `freeze-pN.json`; drafts use `draft-pN.toml`.
|
||||
The freeze digest is SHA-256 of parsed public TOML encoded as UTF-8 JSON with sorted
|
||||
keys, compact separators and `ensure_ascii=False`. Private entries must be represented
|
||||
by opaque digests in that public manifest, as task 3.2 requires.
|
||||
|
||||
The ledger exposes `append(path, row)` and `read(path)`. Rows contain `slice`, `decision`
|
||||
(an opaque decision or experiment label), `purpose`, ISO `date`, `result_digest`, and
|
||||
`freeze_digest`. Appends lock and validate the existing chain, reject duplicates or a
|
||||
changed freeze, and reject qualification of an informed slice for the same decision.
|
||||
Keep the published final digest to authenticate history: the chain detects row edits,
|
||||
but cannot independently prove that an entire history was not replaced or truncated.
|
||||
|
||||
## Sourcing and extraction (tasks 2.1–2.4)
|
||||
|
||||
The coordinator's networked task **2.5** is now:
|
||||
|
||||
```sh
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH=$PWD/src
|
||||
python -m benchmarks.unseen.draft --root "$PWD" --s3-store s3:// --seed 20260601 \
|
||||
--cache /home/mc/ousast-results/plane/unseen-draft --max-hours 8
|
||||
```
|
||||
|
||||
Export `GH_TOKEN` first, or supply it through the root `.env` (exported values win).
|
||||
The same read-only local/S3 stores, `--pointer-manifest`, and explicit `--known-empty`
|
||||
options as task 1.5 apply. Missing or unreadable sources fail. The used set is rebuilt
|
||||
and canonicalized before candidate selection, including redirects and fork networks.
|
||||
No population v3 file is opened and no guard/bisection is run by this command; that is
|
||||
reserved for group 3. An existing draft is never overwritten.
|
||||
|
||||
The command writes `--cache/journal/draft-p1.toml` and
|
||||
`--cache/journal/private-draft-p1.toml` only after all sourcing stages complete.
|
||||
Both stay outside the repository, with mode 0600. The public-format file contains permissive entries
|
||||
and, for every private entry, only an opaque ID, slice, private marker, and SHA-256
|
||||
of canonical JSON containing its URL and ordered `[base, head]` pairs. Private entries
|
||||
carry the complete records, including licences, in the private-format file.
|
||||
Licensing is re-read through GitHub's licence endpoint **at the actual extracted
|
||||
vulnerability replay head**; the candidate's earlier fix-parent classification is
|
||||
provisional. An explicit SPDX declaration in the pinned file takes precedence over
|
||||
the API's classification. Missing/unknown/compound licences remain private.
|
||||
|
||||
Three seeded slices each contain 100 repositories, one vulnerability range and twelve
|
||||
ordinary changes per repository. Each ecosystem/post-cutoff stratum differs from its
|
||||
proportional per-slice target by less than one repository. Post-cutoff candidates are
|
||||
attempted first within each ecosystem. Below 300 eligible repositories the command
|
||||
exits 1 with counts and does not write a partial draft or relax any selection bar.
|
||||
The group-2 measured exit and label confirmation remain coordinator tasks 2.5/2.6;
|
||||
passing implementation tests does not assert that 300 eligible repositories exist.
|
||||
|
||||
Clones and checkpoints default to `results_root()/unseen-draft` (normally
|
||||
`~/ousast-results/plane/unseen-draft`), overridable by `--cache /outside/path`.
|
||||
The cache cannot lie inside `--root`. One opaque, disposable clone exists at a time;
|
||||
it is removed on success, rejection, or failure. A full-objects `--no-checkout` clone
|
||||
brings the whole commit graph and every blob in one pack, so temporal thirds, blame,
|
||||
and release-tag ancestry read locally; `--no-checkout` still skips the working tree.
|
||||
This is a full clone, not a shallow history, and has no blob:none promisor, so a
|
||||
genuinely missing object errors loudly instead of silently round-tripping (the old
|
||||
promisor path fetched blobs one at a time, hundreds of round-trips per repo). The
|
||||
500 MiB disk cap is checked after Git commands; one in-flight command can transiently
|
||||
exceed it before rejection and cleanup.
|
||||
Git output is captured, never forwarded. Each repository must read a nonempty source
|
||||
blob before extraction. Summary instrumentation includes source read operations and
|
||||
actual bytes, clone count, cumulative/peak disk bytes, wall time, and logical API calls
|
||||
and HTTP attempts. Zero source bytes or a failed Git read is an instrument failure,
|
||||
not an eligibility drop. Stdout is a JSON summary of counts/digests only; no names or
|
||||
URLs enter stdout or a measurement record. Raw per-change data stays in the external journal and manifests.
|
||||
|
||||
Extraction uses Git and syntax boundaries, with no scanner, engine, or model. Python
|
||||
uses stdlib AST; Java/PHP/JS/TS use conservative named-function brace matching. Ambiguous
|
||||
or unparseable functions can reject a candidate. The first removed code line per
|
||||
fix-touched function is the SZZ sink proxy; pure-addition fixes use its declaration.
|
||||
Normalized text absence, non-root/non-merge ancestry, file/line limits, and the OSV
|
||||
release bracket are checked before accepting `introducing`. Matching tags are `VERSION`
|
||||
or `vVERSION`; an absent/ambiguous tag or missing prior release cannot certify the
|
||||
bracket. After at most five walk-backs, `last_touch` uses `git log -L` with the resolved
|
||||
numeric function extent (also works where Git lacks a language-specific function driver).
|
||||
Method, bias, fallback reasons, head function/range, files/hunks, family, advisory, and
|
||||
cutoff flag are recorded. Task 2.6 must review every third `introducing` label.
|
||||
|
||||
All reviewed advisories are enumerated to exclude **all** known repository fixes from
|
||||
ordinary changes, including old advisories and advisories outside the four candidate
|
||||
ecosystems. Fix URLs are canonicalized before indexing so renames cannot hide them.
|
||||
The ordinary filter uses only the design's narrowed security pattern. It counts
|
||||
rejections (including oversized commits and repositories failing the 40-commit floor),
|
||||
checks both sides for touches to known vulnerable functions, and inspects generated
|
||||
headers and inherited `.gitattributes`. Sampling allocates twelve draws proportionally
|
||||
across three history-time thirds and four source-size buckets, with one per third.
|
||||
|
||||
For offline tests, inject GitHub/OSV clients, the clone context, and `UsedSet` into
|
||||
`draft.run`; HTTP transports and Git subprocess runners also accept fakes. Tests use
|
||||
sanitized REST-shaped data and a temporary synthetic local Git history. No implementation
|
||||
test contacts GitHub, OSV, a model, or a cluster. HTTP retries are bounded to five attempts;
|
||||
rate-limit reset/Retry-After waits and transient 5xx backoff are tested with fake clocks
|
||||
and sleepers. A stale candidate 404 is counted; other service failures fail the run.
|
||||
|
||||
### Checkpoint and resume
|
||||
|
||||
The command above uses journal directory
|
||||
`/home/mc/ousast-results/plane/unseen-draft/journal`. Run the same command to resume.
|
||||
`--max-hours` is a per-invocation budget; changing it does not invalidate checkpoints.
|
||||
Create `journal/STOP` to stop between units, then remove it before resuming. A page,
|
||||
repository resolution, candidate decision, or repository extraction already running
|
||||
finishes first, so the time budget is not a hard timeout on a large repository.
|
||||
Clean stops exit 0 with `status: stopped`; failures and shortfalls exit 1. Completion
|
||||
exits 0 with `status: draft`. No partial manifest is published.
|
||||
|
||||
`inputs.json` binds the seed, source-labelled used-set digest, known-empty declarations,
|
||||
clone cap, and design fingerprints (sourcing/extraction/eligibility/publication code and
|
||||
family taxonomy). A mismatch refuses resume and prints only the changed parameter
|
||||
labels. Use a new external cache for a deliberately changed design or population.
|
||||
The used set is rebuilt each invocation; repository resolutions, including 404s, are
|
||||
reused from the journal. Live GitHub metadata for unfinished units can still change;
|
||||
completed pages and decisions are the durable sourcing snapshot.
|
||||
|
||||
Each completed unit is appended to its stage JSONL and fsynced before work advances:
|
||||
`advisories.jsonl` (pages), `resolution.jsonl` (alias/fork metadata),
|
||||
`eligibility.jsonl` (candidate decisions), `extraction.jsonl` (repository/advisory
|
||||
results, vulnerable range, ordinary draws, pinned licence, method and counts), and
|
||||
`completion.jsonl` (final manifest digest). A lock permits only one writer.
|
||||
`progress.json` is atomically rewritten after every unit with stage counts, last unit,
|
||||
last observed GitHub rate-limit headers, and cumulative active elapsed seconds.
|
||||
These files may contain identities and must stay outside version control. Stdout and
|
||||
repository records contain only counts, digests and fixed labels.
|
||||
|
||||
Replaying completed units makes no GitHub/OSV or clone calls. Final manifests are
|
||||
reconstructed deterministically from journal results; an interrupted publication can
|
||||
be retried, but a different existing manifest is refused. Completed JSONL rows are
|
||||
never rewritten. Malformed or torn records fail closed instead of being interpreted
|
||||
as completed work. Tests interrupt each stage with fakes, then compare resumed and
|
||||
uninterrupted manifests byte for byte.
|
||||
|
||||
### Planning estimate for task 2.5 (not measured)
|
||||
|
||||
Let A be all reviewed advisories, R distinct fix-linked repository identities, U used
|
||||
identities not already cached, C candidate advisories passing early filters, and K
|
||||
repositories surviving extraction. GitHub requests are approximately
|
||||
`ceil(A/100) + R + U + 2*C + K` (enumeration, canonical repositories, fix metadata,
|
||||
provisional licence, final licence), plus retries. OSV requests are at most C. Git
|
||||
fetches and lazy blob transfers are additional network operations, not GitHub API calls.
|
||||
|
||||
For a planning scenario of A=40,000, R=15,000, U≈1,000, C=600–1,500, K=300:
|
||||
about **18,000–20,000 GitHub calls** and **600–1,500 OSV calls**. The all-advisory index
|
||||
makes this substantially more than 300 repository lookups. At an assumed 5,000-request
|
||||
hourly allowance, allow several rate-limit windows. Actual token limits can differ.
|
||||
For 300–1,000 attempted clones averaging 20–100 MB after source hydration, plan roughly
|
||||
**6–100 GB transferred**, with one clone occupying about 500 MiB at most between commands
|
||||
plus the retained journal and manifests after completion. Transfer volume is an estimate; disk bytes
|
||||
are only a proxy. A working planning range is **8–48 hours**, potentially longer on
|
||||
large histories: every first-parent commit is inspected and lazy blobs can dominate.
|
||||
These are assumptions, not evidence from a network run. The command reports actual
|
||||
counts and wall time; task 2.5 establishes the measured estimate.
|
||||
|
||||
## Dispatcher, replay and scoring (groups 4–6)
|
||||
|
||||
`benchmarks.ax.batch` owns Task transport and the shared scheduler. New workloads supply
|
||||
`Item` input bytes/commands and a `Workload` image, URL allowlist, deadline and result
|
||||
validator. Two AX slots and one VM slot are the default; sandbox death gets one fresh
|
||||
Task before VM fallback, OOM goes straight to VM, and instrument failures get one rerun.
|
||||
Cleanup failure prevents further launches. Create `STOP` in the output directory to stop
|
||||
launching; remove it and repeat the command to resume matching input digests. The legacy
|
||||
engine adapter retains its `questions.json` object name for compatibility; replay inputs
|
||||
and outputs use `.dat`. Both use the same object polling and cleanup implementation.
|
||||
|
||||
Replay reads `pool-pN.toml`, verifies its adjacent `freeze-pN.json` canonical digest,
|
||||
selects `--slice`, and resolves private pointers through `private/pool-pN.toml` (or
|
||||
`--private-manifest`). It never reads a draft. `--dry-run` prints only counts and digests.
|
||||
The five-change proof uses a separate frozen JSON manifest from this project's own
|
||||
source-changing history. Raw output must be outside the repository.
|
||||
|
||||
Each output root contains `inputs.json` and digest-keyed records and artifact directories.
|
||||
The scorer accepts that root directly, joins change labels, and includes missing or stale
|
||||
results as instrument failures. It refuses a slice above 5% failure after reruns. Example:
|
||||
|
||||
```sh
|
||||
python -m benchmarks.unseen.score "$HOME/ousast-results/unseen/p1/1/default" \
|
||||
--out "$HOME/ousast-results/unseen/p1/1/default-score.json"
|
||||
```
|
||||
|
||||
For paired binary arms add `--against BASELINE_ROOT --slice-id p1/2`; `--informed` forces
|
||||
`unseen.gate=not_run`. The later experiment integration owns ledger verification; callers
|
||||
of `gate` must pass that ledger decision. Score-bearing arms use `budgets` with fixed
|
||||
thresholds and `gate(..., binary=False, threshold=..., threshold_fixed=True)`. In-slice
|
||||
thresholds are exploratory and cannot qualify adoption. JVM wall time is explicitly the
|
||||
sum of the hook artifact's scan build/query times, not a separately observed process timer.
|
||||
|
||||
### Coordinator task 4.4
|
||||
|
||||
Run from the merged repository root. Set `ENGINE_IMAGE` to the **engine** digest printed
|
||||
by Engine image CI, `ENGINE_UNITS` and `ENGINE_EXAMPLES` to the frozen input files for the
|
||||
two stored pins, `ENGINE_BASELINE` to their stored record directory, and `AX_KUBECONFIG`
|
||||
to the operator's kube-ax kubeconfig. Host store credentials stay on the VM; use the public
|
||||
HTTPS `S3_ENDPOINT=https://files.because-security.com` and the operator's `OUSAST_MEMORY`.
|
||||
Use a fresh output directory, so the smoke executes instead of resuming prior records.
|
||||
|
||||
```sh
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH=$PWD/src
|
||||
export S3_ENDPOINT=https://files.because-security.com
|
||||
python benchmarks/learn/engine_trace.py \
|
||||
--units "$ENGINE_UNITS" --examples "$ENGINE_EXAMPLES" \
|
||||
--executor ax --parallel 2 --limit 2 --image "$ENGINE_IMAGE" \
|
||||
--kubeconfig "$AX_KUBECONFIG" --out "$HOME/ousast-results/unseen/proof-04"
|
||||
python -m benchmarks.unseen.proof engine --image "$ENGINE_IMAGE" \
|
||||
--records "$HOME/ousast-results/unseen/proof-04" --baseline "$ENGINE_BASELINE" \
|
||||
--out "benchmarks/measurements/$(date +%F)-unseen-04-dispatcher/record.json"
|
||||
```
|
||||
|
||||
### Coordinator task 5.4
|
||||
|
||||
Merge before running this proof. `.github/workflows/engine-image.yml` builds the `engine`
|
||||
and `replay` targets on push to main; it publishes `ousast-engine-task` and
|
||||
`ousast-replay-task` separately and prints their digests. Set `REPLAY_IMAGE` to the latter
|
||||
full `ghcr.io/...@sha256:...` reference. No manual image build is needed. Set
|
||||
`KIND_KUBECONFIG` and `AX_KUBECONFIG` to the corresponding AX kubeconfigs; the kind lane
|
||||
uses the identical Task API/manifest, never a Kubernetes Job.
|
||||
|
||||
```sh
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH=$PWD/src
|
||||
export S3_ENDPOINT=https://files.because-security.com
|
||||
python -m benchmarks.unseen.replay --proof-history \
|
||||
--manifest "$HOME/ousast-results/unseen/proof-05-inputs.json"
|
||||
python -m benchmarks.unseen.replay --manifest "$HOME/ousast-results/unseen/proof-05-inputs.json" \
|
||||
--image "$REPLAY_IMAGE" --lane kind --kubeconfig "$KIND_KUBECONFIG" \
|
||||
--out "$HOME/ousast-results/unseen/proof-05-kind"
|
||||
python -m benchmarks.unseen.replay --manifest "$HOME/ousast-results/unseen/proof-05-inputs.json" \
|
||||
--image "$REPLAY_IMAGE" --lane ax --kubeconfig "$AX_KUBECONFIG" \
|
||||
--out "$HOME/ousast-results/unseen/proof-05-ax"
|
||||
python -m benchmarks.unseen.replay --manifest "$HOME/ousast-results/unseen/proof-05-inputs.json" \
|
||||
--image "$REPLAY_IMAGE" --lane docker \
|
||||
--out "$HOME/ousast-results/unseen/proof-05-vm"
|
||||
python -m benchmarks.unseen.proof replay --image "$REPLAY_IMAGE" \
|
||||
--kind-records "$HOME/ousast-results/unseen/proof-05-kind" \
|
||||
--ax-records "$HOME/ousast-results/unseen/proof-05-ax" \
|
||||
--vm-records "$HOME/ousast-results/unseen/proof-05-vm" \
|
||||
--out "benchmarks/measurements/$(date +%F)-unseen-05-replay-task/record.json"
|
||||
```
|
||||
|
||||
The record command fails if fewer than five results exist, identities differ, bytes/JVM
|
||||
proof is missing, or quick-tier findings differ. It does not convert a quick-only run
|
||||
without a JVM into an engine proof. Pool replays use `--lane mixed` (default) for the two
|
||||
AX slots and single VM overflow/fallback lane; `--arm long_deadline` changes only the
|
||||
hook deadline to 300 seconds and is a separate input digest.
|
||||
|
||||
### Deadline calibration and native agreement
|
||||
|
||||
The 2026-10-05 maintainer decision supersedes the unscaled cluster deadline above.
|
||||
gVisor snapshot preparation can consume 30 seconds before any analysis. Calibrate
|
||||
the pinned replay image against the VM's native Docker **runc** runtime first.
|
||||
The default deterministic sample is the first 20 changes ordered by SHA-256 of
|
||||
change ID; a smaller manifest (including the five-change proof) uses all changes.
|
||||
Sampling never depends on manifest order.
|
||||
|
||||
`calibrate` runs both lanes with a 600-second hook deadline (override `--deadline`
|
||||
if stages still time out). It reads resolution, provenance, preparation, snapshot,
|
||||
and quick timings from the push artifact; snapshot is nested in preparation.
|
||||
`quick_total` sums resolution + provenance + preparation + quick, without counting
|
||||
snapshot twice. Engine build and query times sum across the hook's scans. Hook
|
||||
wall time comes from the instrument block. Missing/nonpositive timings, deadline
|
||||
exhaustion, degraded engine scans or failed instruments prevent a successful
|
||||
calibration; failed changes are never silently dropped. Each factor is the median
|
||||
of paired cluster/native ratios with a deterministic 2,000-resample percentile
|
||||
bootstrap 95% interval. The measurement record contains opaque change digests,
|
||||
counts and timings, with no repository names. Raw artifacts remain outside the repo.
|
||||
|
||||
Set `REPLAY_IMAGE` to the newly built replay image's pinned digest and
|
||||
`REPLAY_IMAGE_BYTES` to an **unpacked image size upper bound in bytes** from the
|
||||
image build. Do not use compressed registry layer size. Docker refuses to pull
|
||||
without this bound or when `df` at its data root reports less than that bound plus
|
||||
1 GiB free. It then explicitly pulls the image before issuing presigned task URLs.
|
||||
Supply `--docker-image-bytes "$REPLAY_IMAGE_BYTES"` to VM and mixed replay commands,
|
||||
including the earlier VM proof command.
|
||||
Launch/pull failures retain exit codes and the last ten sanitized stderr lines in
|
||||
`docker-error.json`, the result record and attempt history. URLs, known environment
|
||||
secrets and credential assignments are redacted. The original VM failure's cause
|
||||
remains unconfirmed until this diagnostic is captured; the old transport discarded it.
|
||||
|
||||
Coordinator commands (these run networked workloads; offline tests use fakes):
|
||||
|
||||
```sh
|
||||
export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
|
||||
export PYTHONPATH=$PWD/src
|
||||
export S3_ENDPOINT=https://files.because-security.com
|
||||
CALIBRATION_RECORD="benchmarks/measurements/$(date +%F)-unseen-calibration/record.json"
|
||||
python -m benchmarks.unseen.calibrate \
|
||||
--manifest "$HOME/ousast-results/unseen/proof-05-inputs.json" \
|
||||
--sample 20 --deadline 600 --image "$REPLAY_IMAGE" \
|
||||
--docker-image-bytes "$REPLAY_IMAGE_BYTES" --kubeconfig "$AX_KUBECONFIG" \
|
||||
--out "$HOME/ousast-results/unseen/calibration" --record "$CALIBRATION_RECORD"
|
||||
|
||||
python -m benchmarks.unseen.replay --manifest benchmarks/unseen/pool-p1.toml \
|
||||
--pool p1 --slice 1 --arm default --lane ax --image "$REPLAY_IMAGE" \
|
||||
--calibration "$CALIBRATION_RECORD" --native-sample 100 \
|
||||
--docker-image-bytes "$REPLAY_IMAGE_BYTES" --kubeconfig "$AX_KUBECONFIG" \
|
||||
--out "$HOME/ousast-results/unseen/p1/1/calibrated-default"
|
||||
python -m benchmarks.unseen.score "$HOME/ousast-results/unseen/p1/1/calibrated-default" \
|
||||
--out "$HOME/ousast-results/unseen/p1/1/calibrated-default-score.json"
|
||||
```
|
||||
|
||||
For a 20-change slice calibration, replace the proof manifest with
|
||||
`benchmarks/unseen/pool-p1.toml --slice 1`; both commands accept `--private-manifest`.
|
||||
For the long arm use `--arm long_deadline` and a separate output root. Calibration
|
||||
requires the same image digest as replay. An explicit `--deadline-scale FACTOR`
|
||||
is an alternative to `--calibration`, not an additional multiplier.
|
||||
|
||||
Cluster deadlines are `round(30 * factor)` or `round(300 * factor)` seconds;
|
||||
native deadlines remain 30 or 300, including VM fallback in mixed runs.
|
||||
Records carry configured and applied scale, base/applied deadline and actual lane;
|
||||
the scorer reports their counts. Task, subprocess and presigned-object budgets
|
||||
grow with the applied deadline.
|
||||
|
||||
Use `--lane ax` (or `kind`) with `--native-sample K` for an agreement run, so every
|
||||
sampled change has a cluster counterpart. The native sample lives in `OUT/native`;
|
||||
the scorer detects it automatically, or accepts `--native PATH`. Missing, failed,
|
||||
wrong-lane or mismatched-image/arm pairs are unknown, never quiet agreements.
|
||||
The decision is whether any of the three finding streams flags the push, not
|
||||
whether the finding matches the vulnerability label.
|
||||
|
||||
**Decision rule:** if the Wilson 95% agreement lower bound is below 95%, do not use
|
||||
the cluster results for the baseline. The score sets `flagged=true`,
|
||||
`baseline_eligible=false` and the adoption gate to `not_run`. Unknown pairs also
|
||||
flag the run. Disagreements are listed by change ID only, alongside per-change
|
||||
flagged/not-flagged decisions. Even perfect agreement needs at least 73 pairs to
|
||||
clear this bound; the five-change proof or a 20-change sample cannot qualify it.
|
||||
@@ -1,267 +0,0 @@
|
||||
"""Durable, content-addressed draw decisions; object I/O stays on the coordinator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
CORPUS_KEYS = ("CORPUS_S3_ENDPOINT", "CORPUS_S3_BUCKET", "CORPUS_AWS_ACCESS_KEY_ID", "CORPUS_AWS_SECRET_ACCESS_KEY")
|
||||
|
||||
|
||||
def corpus_settings(environ: Mapping[str, str] | None = None) -> dict:
|
||||
"""Read corpus-only credentials, naming missing settings without exposing values."""
|
||||
if environ is None:
|
||||
from openultrasast.config import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
environ = os.environ
|
||||
missing = [key for key in CORPUS_KEYS if not environ.get(key)]
|
||||
if missing:
|
||||
raise ValueError(f"Corpus bank needs {', '.join(missing)} in .env or the environment")
|
||||
endpoint = environ["CORPUS_S3_ENDPOINT"].strip()
|
||||
try:
|
||||
parsed = urlsplit(endpoint)
|
||||
valid_endpoint = parsed.scheme in ("http", "https") and bool(parsed.netloc)
|
||||
except ValueError:
|
||||
valid_endpoint = False
|
||||
if not valid_endpoint:
|
||||
raise ValueError("CORPUS_S3_ENDPOINT must be a URL with a scheme (https://host[:port])")
|
||||
settings = {
|
||||
"endpoint": endpoint,
|
||||
"bucket": environ["CORPUS_S3_BUCKET"],
|
||||
"access_key": environ["CORPUS_AWS_ACCESS_KEY_ID"],
|
||||
"secret_key": environ["CORPUS_AWS_SECRET_ACCESS_KEY"],
|
||||
}
|
||||
if environ.get("CORPUS_S3_REGION"):
|
||||
settings["region"] = environ["CORPUS_S3_REGION"]
|
||||
return settings
|
||||
|
||||
|
||||
class CorpusObjects:
|
||||
"""Plain S3 put/get/list, with no bucket administration or capability probes."""
|
||||
|
||||
def __init__(self, *, endpoint: str, bucket: str, access_key: str, secret_key: str, region: str | None = None):
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
self.bucket = bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region or "us-east-1",
|
||||
# RustFS/MinIO at an IP endpoint needs path-style addressing
|
||||
# (virtual-host bucket.<ip> does not resolve) and SigV4. Verified
|
||||
# against ousast-corpus @ 10.0.0.54:9001.
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
retries={"total_max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
def get(self, key: str) -> tuple[bytes, str] | None:
|
||||
try:
|
||||
result = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except Exception as error:
|
||||
if getattr(error, "response", {}).get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return None
|
||||
raise
|
||||
body = result["Body"]
|
||||
try:
|
||||
return body.read(), result.get("VersionId") or result.get("ETag", "")
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
def put(self, key: str, data: bytes, labels: dict) -> None:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType="application/json",
|
||||
Tagging=urlencode(labels),
|
||||
)
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=self.bucket, Prefix=prefix)
|
||||
return [item["Key"] for page in pages for item in page.get("Contents", [])]
|
||||
|
||||
|
||||
def _retry(operation, *args, **kwargs):
|
||||
"""Retry one object operation once; a second failure is fatal to durability."""
|
||||
try:
|
||||
return operation(*args, **kwargs)
|
||||
except Exception:
|
||||
return operation(*args, **kwargs)
|
||||
|
||||
|
||||
class CorpusBank:
|
||||
def __init__(self, client, *, seed: int, prefix: str = "unseen"):
|
||||
self.client = client
|
||||
self.seed = seed
|
||||
self.decisions_prefix = f"{prefix}/{seed}/decisions/"
|
||||
self.resolved_prefix = f"{prefix}/{seed}/resolved/"
|
||||
self.advisories_key = f"{prefix}/{seed}/advisories/list.json"
|
||||
self.repositories_prefix = f"{prefix}/{seed}/repositories/"
|
||||
# get_repo metadata is seed-independent GitHub data: share it across seeds
|
||||
# so the ~thousands of used-set canonicalization lookups are paid once ever.
|
||||
self.repos_prefix = f"{prefix}/repos/"
|
||||
|
||||
@staticmethod
|
||||
def advisory_key(advisory_id: str) -> str:
|
||||
return hashlib.sha256(advisory_id.encode()).hexdigest()
|
||||
|
||||
def _put(self, key: str, body: dict, *, labels: dict) -> None:
|
||||
existing = _retry(self.client.get, key)
|
||||
if existing is not None:
|
||||
previous = json.loads(existing[0])
|
||||
if {field: value for field, value in previous.items() if field != "ts"} == body:
|
||||
return
|
||||
body = {**body, "ts": datetime.now(UTC).isoformat()}
|
||||
data = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
_retry(self.client.put, key, data, labels=labels)
|
||||
|
||||
def bank_resolution(self, advisory_id: str, result: dict, *, contract: str) -> None:
|
||||
decision = "candidate" if "candidate" in result else "reject"
|
||||
self._put(
|
||||
self.resolved_prefix + self.advisory_key(advisory_id) + ".json",
|
||||
{
|
||||
"advisory": advisory_id,
|
||||
"decision": decision,
|
||||
"result": result,
|
||||
"reason": result.get("rejected"),
|
||||
"resolution_contract": contract,
|
||||
"seed": self.seed,
|
||||
},
|
||||
labels={"decision": decision, "contract": contract},
|
||||
)
|
||||
|
||||
def _listed(self, prefix: str, identity: str) -> dict[str, dict]:
|
||||
results = {}
|
||||
for key in sorted(self.client.keys(prefix)):
|
||||
stored = _retry(self.client.get, key)
|
||||
if stored is None:
|
||||
raise ValueError("corpus_resolution_missing")
|
||||
body = json.loads(stored[0])
|
||||
name = body[identity]
|
||||
if key != prefix + self.advisory_key(name) + ".json" or body["seed"] != self.seed:
|
||||
raise ValueError("corpus_resolution_identity")
|
||||
results[name] = body
|
||||
return results
|
||||
|
||||
def resolved(self) -> dict[str, dict]:
|
||||
results = self._listed(self.resolved_prefix, "advisory")
|
||||
for body in results.values():
|
||||
expected = "candidate" if "candidate" in body["result"] else "reject"
|
||||
if body["decision"] != expected:
|
||||
raise ValueError("corpus_resolution_invalid")
|
||||
return results
|
||||
|
||||
def cache_advisories(self, rows: list, *, contract: str) -> None:
|
||||
if self.cached_advisories(contract=contract) is not None:
|
||||
return
|
||||
self._put(
|
||||
self.advisories_key,
|
||||
{"advisories": rows, "advisories_contract": contract, "seed": self.seed},
|
||||
labels={"contract": contract},
|
||||
)
|
||||
|
||||
def cached_advisories(self, *, contract: str) -> list | None:
|
||||
stored = _retry(self.client.get, self.advisories_key)
|
||||
if stored is None:
|
||||
return None
|
||||
body = json.loads(stored[0])
|
||||
if body["seed"] != self.seed:
|
||||
raise ValueError("corpus_advisories_identity")
|
||||
return body["advisories"] if body["advisories_contract"] == contract else None
|
||||
|
||||
def bank_repository(self, name: str, result: dict | None, *, contract: str) -> None:
|
||||
"""Retain redirects, fork identity and 404s needed by the all-fix index."""
|
||||
self._put(
|
||||
self.repositories_prefix + self.advisory_key(name) + ".json",
|
||||
{"repository": name, "result": result, "resolution_contract": contract, "seed": self.seed},
|
||||
labels={"contract": contract},
|
||||
)
|
||||
|
||||
def repositories(self, *, contract: str) -> dict[str, dict | None]:
|
||||
return {
|
||||
name: body["result"]
|
||||
for name, body in self._listed(self.repositories_prefix, "repository").items()
|
||||
if body["resolution_contract"] == contract
|
||||
}
|
||||
|
||||
def cache_repo(self, name: str, metadata: dict | None, *, contract: str) -> None:
|
||||
"""Persist one get_repo result (metadata, or None for a 404) by normalized name."""
|
||||
self._put(
|
||||
self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json",
|
||||
{"name": name, "metadata": metadata, "resolution_contract": contract},
|
||||
labels={"contract": contract},
|
||||
)
|
||||
|
||||
def get_cached_repo(self, name: str, *, contract: str) -> tuple[bool, dict | None]:
|
||||
"""Return (found, metadata). found is True even when metadata is None (a cached 404)."""
|
||||
stored = _retry(self.client.get, self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json")
|
||||
if stored is None:
|
||||
return (False, None)
|
||||
body = json.loads(stored[0])
|
||||
if body.get("name") != name or body.get("resolution_contract") != contract:
|
||||
return (False, None)
|
||||
return (True, body["metadata"])
|
||||
|
||||
@staticmethod
|
||||
def key_for(repository: str, fix: str) -> str:
|
||||
return hashlib.sha256(f"{repository}\n{fix}".encode()).hexdigest()
|
||||
|
||||
def bank(self, repository: str, fix: str, result: dict, *, contract: str) -> None:
|
||||
decision = {
|
||||
"repository": repository,
|
||||
"fix": fix,
|
||||
"decision": "accept" if "entry" in result else "reject",
|
||||
"reason": result.get("rejected"),
|
||||
"entry": result.get("entry"),
|
||||
"ordinary_exclusions": result.get("ordinary_exclusions", {}),
|
||||
"counts": result.get("counts", {}),
|
||||
"contract": contract,
|
||||
"seed": self.seed,
|
||||
}
|
||||
key = self.decisions_prefix + self.key_for(repository, fix) + ".json"
|
||||
existing = _retry(self.client.get, key)
|
||||
if existing is not None:
|
||||
previous = json.loads(existing[0])
|
||||
if {field: value for field, value in previous.items() if field != "ts"} == decision:
|
||||
return # Preserve the first timestamp and avoid an unnecessary object version.
|
||||
decision["ts"] = datetime.now(UTC).isoformat()
|
||||
data = json.dumps(decision, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
_retry(self.client.put, key, data, labels={"decision": decision["decision"], "contract": contract})
|
||||
|
||||
def decided(self) -> dict[str, dict]:
|
||||
decisions = {}
|
||||
for key in sorted(self.client.keys(self.decisions_prefix)):
|
||||
stored = _retry(self.client.get, key)
|
||||
if stored is None:
|
||||
raise ValueError("corpus_decision_missing")
|
||||
decision = json.loads(stored[0])
|
||||
digest = self.key_for(decision["repository"], decision["fix"])
|
||||
if key != self.decisions_prefix + digest + ".json" or decision["seed"] != self.seed:
|
||||
raise ValueError("corpus_decision_identity")
|
||||
if decision["decision"] not in {"accept", "reject"}:
|
||||
raise ValueError("corpus_decision_invalid")
|
||||
decisions[digest] = decision
|
||||
return decisions
|
||||
|
||||
|
||||
def split_decided(decisions: Mapping[str, dict], *, contract: str) -> tuple[list[dict], set[str]]:
|
||||
"""Split reusable results, sorted independently of bucket listing order.
|
||||
|
||||
The coordinator merges entries into the full advisory population before seeded
|
||||
ordering: shuffling only the accepted subset would change the draw's order.
|
||||
"""
|
||||
accepted = [row["entry"] for row in decisions.values() if row["decision"] == "accept"]
|
||||
accepted.sort(key=lambda entry: (entry.get("advisory", ""), entry["repository"], entry["fix"]))
|
||||
reusable = {key for key, row in decisions.items() if row["decision"] == "accept" or row["contract"] == contract}
|
||||
return accepted, reusable
|
||||
@@ -1,171 +0,0 @@
|
||||
"""Stage a cache draft, bisect with the black-box guard, and retain a digest receipt.
|
||||
|
||||
Coordinator: python -m benchmarks.unseen.bisect_guard --draft CACHE/draft-p1.toml
|
||||
The cache is never changed. Successful subsets remain at benchmarks/unseen/draft-p1.toml
|
||||
and private/draft-p1.toml; failures remove only this invocation's staging files.
|
||||
Stdout reports only the dropped count. No reservation is opened by this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from .draft import manifest, private_digest
|
||||
from .extract import InstrumentFailure
|
||||
from .journal import atomic, digest
|
||||
from .source import PERMISSIVE
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> dict:
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
if not raw:
|
||||
raise ValueError
|
||||
data = tomllib.loads(raw.decode("utf-8"))
|
||||
if set(data) - {"status", "pool", "seed", "repository"}:
|
||||
raise ValueError
|
||||
if data["status"] != "draft" or data["pool"] != "p1" or type(data["seed"]) is not int:
|
||||
raise ValueError
|
||||
if not isinstance(data.get("repository", []), list):
|
||||
raise ValueError
|
||||
return data
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
raise InstrumentFailure("invalid_draft") from None
|
||||
|
||||
|
||||
def read_pair(public_path: Path, private_path: Path) -> tuple[dict, dict]:
|
||||
"""Validate the licensing split and every private pointer before staging names."""
|
||||
public, private = read_manifest(public_path), read_manifest(private_path)
|
||||
try:
|
||||
if public["seed"] != private["seed"]:
|
||||
raise ValueError
|
||||
hidden = {row["id"]: row for row in private.get("repository", [])}
|
||||
rows = public.get("repository", [])
|
||||
if len(hidden) != len(private.get("repository", [])) or len({r["id"] for r in rows}) != len(rows):
|
||||
raise ValueError
|
||||
if set(hidden) != {r["id"] for r in rows if r.get("private")}:
|
||||
raise ValueError
|
||||
# Slice IDs come from the draft; bisect may leave sparse slices.
|
||||
for row in rows:
|
||||
if type(row["slice"]) is not int or row["slice"] < 1:
|
||||
raise ValueError
|
||||
if row.get("private"):
|
||||
full = hidden[row["id"]]
|
||||
expected = {"id": full["id"], "slice": full["slice"], "private": True, "sha256": private_digest(full)}
|
||||
if row != expected or full["license_class"] != "private":
|
||||
raise ValueError
|
||||
elif row["license_class"] != "permissive" or row["license_spdx"] not in PERMISSIVE:
|
||||
raise ValueError
|
||||
return public, private
|
||||
except (ValueError, KeyError, TypeError):
|
||||
raise InstrumentFailure("invalid_draft_pair") from None
|
||||
|
||||
|
||||
def pytest_guard(root: Path) -> int:
|
||||
"""Read only the exit code; never capture, inspect or forward either stream."""
|
||||
try:
|
||||
return subprocess.run(
|
||||
[".venv/bin/pytest", "-q", "-p", "no:cacheprovider", "tests/test_independent_population.py", "-k", "referenced"],
|
||||
cwd=root,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
).returncode
|
||||
except OSError:
|
||||
raise InstrumentFailure("guard_launch") from None
|
||||
|
||||
|
||||
def run(root: Path, draft: Path, *, private_draft: Path | None = None, guard_runner=None) -> dict:
|
||||
"""Inject guard_runner(root) -> exit code for offline tests; default is pytest_guard.
|
||||
|
||||
Run from an otherwise clean reservation directory. Existing staging/receipt or a
|
||||
frozen p1 is refused, so a crash cannot silently turn a partial subset into input.
|
||||
The receipt binds both retained manifests; freeze refuses edits after the guard.
|
||||
"""
|
||||
private_draft = private_draft or draft.with_name("private-draft-p1.toml")
|
||||
public, private = read_pair(draft, private_draft)
|
||||
rows = public.get("repository", [])
|
||||
if not rows:
|
||||
raise InstrumentFailure("empty_draft")
|
||||
directory = root / "benchmarks/unseen"
|
||||
public_path = directory / "draft-p1.toml"
|
||||
private_path = directory / "private/draft-p1.toml"
|
||||
receipt_path = directory / "bisect-p1.json"
|
||||
targets = (private_path, public_path, receipt_path)
|
||||
if any(p.exists() for p in (*targets, directory / "pool-p1.toml", directory / "freeze-p1.json")):
|
||||
raise FileExistsError("staging_exists")
|
||||
private_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
runner = guard_runner or pytest_guard
|
||||
created = []
|
||||
|
||||
def check(subset):
|
||||
ids = {r["id"] for r in subset}
|
||||
selected = [r for r in private.get("repository", []) if r["id"] in ids]
|
||||
atomic(private_path, manifest(selected, private["seed"]).encode("utf-8"))
|
||||
atomic(public_path, manifest(subset, public["seed"]).encode("utf-8"))
|
||||
try:
|
||||
code = runner(root)
|
||||
except Exception:
|
||||
raise InstrumentFailure("guard_launch") from None
|
||||
if type(code) is not int or code not in (0, 1):
|
||||
raise InstrumentFailure("guard_exit")
|
||||
return code
|
||||
|
||||
def retain(subset):
|
||||
if not subset or check(subset) == 0:
|
||||
return subset
|
||||
if len(subset) == 1:
|
||||
return []
|
||||
middle = len(subset) // 2
|
||||
return retain(subset[:middle]) + retain(subset[middle:])
|
||||
|
||||
try:
|
||||
# Reserve destinations exclusively before any rewrites; never truncate an
|
||||
# existing coordinator's files. Empty manifests establish the control.
|
||||
for path in targets:
|
||||
with path.open("xb") as handle:
|
||||
created.append(path)
|
||||
os.fchmod(handle.fileno(), 0o600)
|
||||
if check([]) != 0:
|
||||
raise InstrumentFailure("guard_baseline")
|
||||
kept = retain(rows)
|
||||
if check(kept) != 0:
|
||||
raise InstrumentFailure("guard_final")
|
||||
result = {
|
||||
"guard_result": 0,
|
||||
"dropped_count": len(rows) - len(kept),
|
||||
"input_count": len(rows),
|
||||
"draft_digest": digest(read_manifest(public_path)),
|
||||
"private_draft_digest": digest(read_manifest(private_path)),
|
||||
}
|
||||
atomic(receipt_path, (json.dumps(result, sort_keys=True) + "\n").encode())
|
||||
return result
|
||||
except BaseException:
|
||||
for path in reversed(created):
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--draft", type=Path, required=True)
|
||||
parser.add_argument("--private-draft", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
result = run(args.root, args.draft, private_draft=args.private_draft)
|
||||
except Exception:
|
||||
print(json.dumps({"status": "instrument_failure"}))
|
||||
return 1
|
||||
print(json.dumps({"dropped_count": result["dropped_count"]}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,128 +0,0 @@
|
||||
"""Paired native/cluster deadline calibration; publish only opaque IDs and numbers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
import statistics
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import write_json
|
||||
from benchmarks.unseen.replay import canonical, load_slice, run_replays, sample_changes
|
||||
from benchmarks.unseen.score import coverage, load_records
|
||||
|
||||
|
||||
def stage_times(record):
|
||||
if coverage(record) == "instrument_failure":
|
||||
raise ValueError("calibration instrument failed")
|
||||
push = record["push"]
|
||||
if "deadline_exhausted" in json.dumps(push) or any(t.get("status") == "failed" for t in push.get("quick_tier", [])):
|
||||
raise ValueError("calibration deadline exhausted or quick tier failed; increase deadline")
|
||||
raw = push.get("timings", {})
|
||||
timings = {}
|
||||
for key, value in raw.items():
|
||||
key = re.sub(r"^comparison_\d+_", "", key)
|
||||
if not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
||||
raise ValueError("invalid stage timing")
|
||||
timings[key] = timings.get(key, 0) + value
|
||||
required = ("resolution_seconds", "provenance_seconds", "preparation_seconds", "quick_seconds", "snapshot_seconds")
|
||||
if any(key not in timings for key in required):
|
||||
raise ValueError("missing completed quick-stage timings")
|
||||
scans = [r["scan"] for r in push.get("scans", [])]
|
||||
if any(s.get("degradations") for s in scans):
|
||||
raise ValueError("incomplete engine stages; calibration cannot use degraded scans")
|
||||
if not scans or any(k not in s for s in scans for k in ("build_seconds", "query_seconds")):
|
||||
raise ValueError("missing engine timings")
|
||||
result = {
|
||||
"snapshot": timings["snapshot_seconds"], # Nested inside preparation; never counted twice.
|
||||
"preparation": timings["preparation_seconds"],
|
||||
"quick": timings["quick_seconds"],
|
||||
"quick_total": sum(timings[k] for k in required if k != "snapshot_seconds"),
|
||||
"engine_build": sum(s["build_seconds"] for s in scans),
|
||||
"engine_queries": sum(s["query_seconds"] for s in scans),
|
||||
"hook": record["instrument"].get("timings", {}).get("pre-push"),
|
||||
}
|
||||
if any(not isinstance(v, (int, float)) or not math.isfinite(v) or v <= 0 for v in result.values()):
|
||||
raise ValueError("missing or nonpositive stage timing")
|
||||
return result
|
||||
|
||||
|
||||
def slowdown(native, cluster, *, seed=0, draws=2000):
|
||||
if not native or len(native) != len(cluster):
|
||||
raise ValueError("calibration needs paired timings")
|
||||
if any(not math.isfinite(v) or v <= 0 for v in [*native, *cluster]):
|
||||
raise ValueError("timings must be finite and positive")
|
||||
ratios = [b / a for a, b in zip(native, cluster, strict=True)]
|
||||
rng = random.Random(seed)
|
||||
bootstrap = sorted(statistics.median(rng.choices(ratios, k=len(ratios))) for _ in range(draws))
|
||||
return {
|
||||
"n": len(ratios),
|
||||
"factor": statistics.median(ratios),
|
||||
"bootstrap95": [bootstrap[int(0.025 * (draws - 1))], bootstrap[int(0.975 * (draws - 1))]],
|
||||
}
|
||||
|
||||
|
||||
def calibration_record(native, cluster):
|
||||
nn, cc = ({r["id"]: r for r in rows} for rows in (native, cluster))
|
||||
if not nn or nn.keys() != cc.keys() or len(nn) != len(native) or len(cc) != len(cluster):
|
||||
raise ValueError("calibration requires exactly paired unique changes")
|
||||
pairs = []
|
||||
for key in sorted(nn):
|
||||
if nn[key].get("lane") != "docker" or cc[key].get("lane") != "ax":
|
||||
raise ValueError("calibration requires native docker and cluster ax lanes")
|
||||
pairs.append(
|
||||
{"change_id": hashlib.sha256(str(key).encode()).hexdigest(), "native": stage_times(nn[key]), "cluster": stage_times(cc[key])}
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"changes": len(pairs),
|
||||
"pairs": pairs,
|
||||
**{
|
||||
stage: slowdown([p["native"][stage] for p in pairs], [p["cluster"][stage] for p in pairs])
|
||||
for stage in ("quick_total", "engine_build", "engine_queries")
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--private-manifest", type=Path)
|
||||
parser.add_argument("--slice", default="1")
|
||||
parser.add_argument("--sample", type=int, default=20)
|
||||
parser.add_argument("--deadline", type=int, default=600)
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--docker-image-bytes", type=int, required=True, help="unpacked image size upper bound")
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--kubeconfig", type=Path)
|
||||
parser.add_argument("--atespace", default="default")
|
||||
parser.add_argument("--out", type=Path, default=Path.home() / "ousast-results/unseen/calibration")
|
||||
parser.add_argument("--record", type=Path, default=Path("benchmarks/measurements") / f"{date.today()}-unseen-calibration/record.json")
|
||||
args = parser.parse_args()
|
||||
if args.deadline < 1 or args.sample < 1:
|
||||
parser.error("deadline and sample must be positive")
|
||||
args.arm = "default"
|
||||
changes = sample_changes(load_slice(args.manifest, args.slice, args.private_manifest), args.sample)
|
||||
failed = False
|
||||
for lane, directory in (("docker", "native"), ("ax", "cluster")):
|
||||
failed |= run_replays(changes, args, args.out / directory, lane, deadline=args.deadline)
|
||||
result = {"status": "failed", "changes": len(changes)}
|
||||
if not failed:
|
||||
try:
|
||||
result = calibration_record(load_records(args.out / "native"), load_records(args.out / "cluster"))
|
||||
except ValueError as exc:
|
||||
result["reason"] = str(exc) # Only locally controlled diagnostics.
|
||||
result.update(
|
||||
image=args.image, deadline_seconds=args.deadline, sample_digest=hashlib.sha256(canonical([c["id"] for c in changes])).hexdigest()
|
||||
)
|
||||
write_json(args.record, result)
|
||||
return 0 if result["status"] == "ok" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,910 +0,0 @@
|
||||
"""Coordinator-only sourcing command: python -m benchmarks.unseen.draft --help.
|
||||
|
||||
All network and git operations are injected at the run() boundary. Only counts,
|
||||
digests and fixed failure codes go to stdout; identity-bearing output stays in the
|
||||
private journal or corpus bucket. No population guard is run here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from collections import Counter, defaultdict, deque
|
||||
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from . import eligibility, extract, source
|
||||
from .bank import CorpusBank, CorpusObjects, corpus_settings, split_decided
|
||||
from .extract import EXTRACTION_CONTRACT, RESOLUTION_CONTRACT, Git, InstrumentFailure
|
||||
from .journal import Journal, ResolvedGitHub, ResumeMismatch, Stopped, atomic, digest, external
|
||||
|
||||
|
||||
def canonical(value) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
|
||||
|
||||
def private_digest(row: dict) -> str:
|
||||
return hashlib.sha256(canonical({"url": row["url"], "commits": [[c["base"], c["head"]] for c in row["changes"]]})).hexdigest()
|
||||
|
||||
|
||||
def validate_pool(pool_size: int, slices: int) -> None:
|
||||
if pool_size < 1:
|
||||
raise ValueError("--pool-size must be at least 1")
|
||||
if slices < 1 or slices > pool_size:
|
||||
raise ValueError("--slices must be between 1 and --pool-size")
|
||||
if pool_size % slices:
|
||||
raise ValueError("--pool-size must be divisible by --slices")
|
||||
|
||||
|
||||
def assign_slices(rows: list[dict], *, seed: int, pool_size: int = 300, slices: int = 3) -> list[dict]:
|
||||
validate_pool(pool_size, slices)
|
||||
if len(rows) != pool_size or len({r["repository"] for r in rows}) != pool_size:
|
||||
raise ValueError("pool_size")
|
||||
for row in rows:
|
||||
kinds = Counter(c["kind"] for c in row["changes"])
|
||||
if kinds != {"vulnerability": 1, "ordinary": 12}:
|
||||
raise ValueError("change_count")
|
||||
if row["license_class"] not in {"permissive", "private"}:
|
||||
raise ValueError("license_class")
|
||||
if row["license_class"] == "permissive" and row["license_spdx"] not in source.PERMISSIVE:
|
||||
raise ValueError("license_class")
|
||||
rng = random.Random(seed)
|
||||
groups = defaultdict(list)
|
||||
for row in sorted(rows, key=lambda r: r["repository"]):
|
||||
groups[row["ecosystem"], row["post_cutoff"]].append(row)
|
||||
sizes = Counter({i: 0 for i in range(1, slices + 1)})
|
||||
assigned = []
|
||||
for key in sorted(groups):
|
||||
group = groups[key]
|
||||
rng.shuffle(group)
|
||||
quotient, remainder = divmod(len(group), slices)
|
||||
# Give each stratum its floor on all slices, then fill globally least-full slices.
|
||||
quotas = {i: quotient for i in sizes}
|
||||
order = list(sizes)
|
||||
rng.shuffle(order)
|
||||
order.sort(key=lambda i: sizes[i])
|
||||
for i in order[:remainder]:
|
||||
quotas[i] += 1
|
||||
offset = 0
|
||||
for slice_id in sorted(quotas):
|
||||
for row in group[offset : offset + quotas[slice_id]]:
|
||||
digest = private_digest(row)
|
||||
assigned.append({**row, "id": "r-" + digest[:24], "slice": slice_id})
|
||||
sizes[slice_id] += quotas[slice_id]
|
||||
offset += quotas[slice_id]
|
||||
if set(sizes.values()) != {pool_size // slices}:
|
||||
raise ValueError("slice_size")
|
||||
return sorted(assigned, key=lambda r: (r["slice"], r["id"]))
|
||||
|
||||
|
||||
def toml(value) -> str:
|
||||
"""Inline TOML values keep nested change records structured without a new dependency."""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, str):
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, list):
|
||||
return "[" + ", ".join(toml(v) for v in value) + "]"
|
||||
if isinstance(value, dict):
|
||||
return "{" + ", ".join(json.dumps(k) + " = " + toml(v) for k, v in sorted(value.items()) if v is not None) + "}"
|
||||
raise ValueError("unsupported_manifest_type")
|
||||
|
||||
|
||||
def manifest(rows: list[dict], seed: int) -> str:
|
||||
lines = ['status = "draft"', 'pool = "p1"', f"seed = {seed}"]
|
||||
for row in rows:
|
||||
lines.append("\n[[repository]]")
|
||||
lines.extend(f"{key} = {toml(value)}" for key, value in sorted(row.items()) if value is not None)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_draft(root: Path, assigned: list[dict], *, seed: int) -> tuple[Path, Path]:
|
||||
public_path = root / "benchmarks/unseen/draft-p1.toml"
|
||||
private_path = root / "benchmarks/unseen/private/draft-p1.toml"
|
||||
if public_path.exists() or private_path.exists():
|
||||
raise FileExistsError("draft_exists")
|
||||
public, private = [], []
|
||||
for row in assigned:
|
||||
if row["license_class"] == "permissive":
|
||||
public.append(row)
|
||||
else:
|
||||
private.append(row)
|
||||
public.append({"id": row["id"], "sha256": private_digest(row), "slice": row["slice"], "private": True})
|
||||
private_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
# Exclusive creates protect drafts awaiting human label review. Private first: never
|
||||
# publish a pointer whose private manifest failed to write.
|
||||
with private_path.open("x") as handle:
|
||||
os.chmod(private_path, 0o600)
|
||||
handle.write(manifest(private, seed))
|
||||
with public_path.open("x") as handle:
|
||||
handle.write(manifest(public, seed))
|
||||
return public_path, private_path
|
||||
|
||||
|
||||
class Clones:
|
||||
"""Disposable blobless clones; each worker receives its own counter accumulator."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root: Path,
|
||||
cache: Path,
|
||||
*,
|
||||
runner=None,
|
||||
limit_bytes=500 * 1024 * 1024,
|
||||
deadline_seconds: float | None = None,
|
||||
parse_deadline_seconds: float | None = 120.0,
|
||||
extract_workers: int = 6,
|
||||
):
|
||||
if extract_workers < 1:
|
||||
raise ValueError("invalid_extract_workers")
|
||||
self.extract_workers = extract_workers
|
||||
self.cache = external(root, cache)
|
||||
self.runner = runner or subprocess.run
|
||||
self.limit_bytes = limit_bytes
|
||||
self.deadline_seconds = deadline_seconds
|
||||
self.parse_deadline_seconds = parse_deadline_seconds
|
||||
self.counts = Counter()
|
||||
|
||||
@contextmanager
|
||||
def open(self, url: str):
|
||||
self.cache.mkdir(parents=True, exist_ok=True)
|
||||
started = time.monotonic()
|
||||
with tempfile.TemporaryDirectory(prefix="clone-", dir=self.cache) as directory:
|
||||
path = Path(directory)
|
||||
git = Git(path, runner=self.runner, limit_bytes=self.limit_bytes, deadline_seconds=self.deadline_seconds)
|
||||
try:
|
||||
# Full commit graph is needed for temporal thirds and OSV brackets.
|
||||
# A full-objects clone (no blob:none promisor) brings every blob in
|
||||
# one pack, so blame/log/diff read locally instead of fetching each
|
||||
# blob on demand (the old promisor path cost ~hundreds of round-trips
|
||||
# per repo). --no-checkout still skips the working tree. With no
|
||||
# promisor remote, a genuinely missing object errors loudly rather
|
||||
# than silently round-tripping. Disk is bounded by limit_bytes below.
|
||||
done = self.runner(
|
||||
["git", "-c", "credential.helper=", "clone", "--quiet", "--no-checkout", url, str(path)],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
timeout=git.command_timeout(),
|
||||
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
||||
)
|
||||
if done.returncode:
|
||||
raise InstrumentFailure("clone_failed")
|
||||
self.counts["clones"] += 1
|
||||
if git.disk_bytes() > self.limit_bytes:
|
||||
raise InstrumentFailure("clone_size_cap")
|
||||
yield git
|
||||
except subprocess.TimeoutExpired:
|
||||
raise extract.ExtractionTimeout("extraction_timeout") from None
|
||||
finally:
|
||||
size = git.disk_bytes()
|
||||
self.counts["clone_bytes"] += size
|
||||
self.counts["peak_clone_bytes"] = max(self.counts["peak_clone_bytes"], size)
|
||||
self.counts["source_bytes_read"] += git.bytes_read
|
||||
self.counts["source_files_read"] += git.files_read
|
||||
self.counts["clone_wall_seconds"] += round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def prove_source(git, revision: str) -> dict:
|
||||
files = git.run("ls-tree", "-r", "--name-only", revision).splitlines()
|
||||
for path in files:
|
||||
if Path(path).suffix in extract.LANGUAGES and not extract.generated_path(path):
|
||||
text = git.blob(revision, path)
|
||||
if text:
|
||||
return {"files": 1, "bytes": getattr(git, "last_blob_bytes", len(text.encode()))}
|
||||
raise InstrumentFailure("zero_source_bytes")
|
||||
|
||||
|
||||
class BankedGitHub(ResolvedGitHub):
|
||||
"""Stream the fix index's repository metadata before continuing the draw."""
|
||||
|
||||
def __init__(self, github, journal, bank):
|
||||
super().__init__(github, journal)
|
||||
self.bank = bank
|
||||
self.resolutions = bank.repositories(contract=RESOLUTION_CONTRACT)
|
||||
journal.rows["resolution"].update({name: body["metadata"] if body is not None else None for name, body in self.resolutions.items()})
|
||||
|
||||
def get_repo(self, name):
|
||||
# Bucket-cache get_repo (the universal resolution choke point) so the
|
||||
# ~thousands of used-set canonicalization lookups and candidate resolves
|
||||
# are not re-fetched from GitHub every run. 404s are cached as None too.
|
||||
key = eligibility.normalize(name)
|
||||
if key not in self.journal.rows["resolution"]:
|
||||
found, value = self.bank.get_cached_repo(key, contract=RESOLUTION_CONTRACT)
|
||||
if not found:
|
||||
self.journal.boundary()
|
||||
try:
|
||||
value = self.github.get_repo(key)
|
||||
except eligibility.RepositoryNotFound:
|
||||
value = None
|
||||
self.bank.cache_repo(key, value, contract=RESOLUTION_CONTRACT)
|
||||
self.journal.put("resolution", key, value)
|
||||
value = self.journal.rows["resolution"][key]
|
||||
if value is None:
|
||||
raise eligibility.RepositoryNotFound("not_found")
|
||||
return value
|
||||
|
||||
def resolve(self, name):
|
||||
key = eligibility.normalize(name)
|
||||
if key not in self.resolutions:
|
||||
try:
|
||||
resolved = eligibility.resolve(name, self)
|
||||
except eligibility.RepositoryNotFound:
|
||||
body = None
|
||||
else:
|
||||
body = {
|
||||
"metadata": self.journal.rows["resolution"].get(key),
|
||||
"full_name": resolved.full_name,
|
||||
"names": sorted(resolved.names),
|
||||
"network": resolved.network,
|
||||
}
|
||||
self.bank.bank_repository(key, body, contract=RESOLUTION_CONTRACT)
|
||||
self.resolutions[key] = body
|
||||
body = self.resolutions[key]
|
||||
if body is None:
|
||||
raise eligibility.RepositoryNotFound("not_found")
|
||||
return eligibility.Resolution(body["full_name"], frozenset(body["names"]), body["network"])
|
||||
|
||||
|
||||
def resolve_repository(name, github):
|
||||
if isinstance(github, BankedGitHub):
|
||||
return github.resolve(name)
|
||||
return eligibility.resolve(name, github)
|
||||
|
||||
|
||||
def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
|
||||
"""Include ALL fixes in the enumerated population, even from rejected candidates.
|
||||
|
||||
Canonicalize link names as well, so an old advisory URL survives repository renames.
|
||||
A 404 is not treated as proof that an advisory is unrelated to a selected repository.
|
||||
"""
|
||||
index = defaultdict(set)
|
||||
for row in rows:
|
||||
for name, sha in source.fix_links(row):
|
||||
try:
|
||||
resolved = resolve_repository(name, github)
|
||||
except eligibility.RepositoryNotFound:
|
||||
index[name].add(sha)
|
||||
continue
|
||||
index[resolved.full_name].add(sha)
|
||||
return dict(index)
|
||||
|
||||
|
||||
def repository_changes(git, candidate, fixes: set[str], *, seed: int):
|
||||
resolved = []
|
||||
for fix in sorted(fixes | {candidate.fix}):
|
||||
try:
|
||||
if fix != candidate.fix:
|
||||
# Extra advisory fixes are best-effort exclusions. The candidate
|
||||
# was fetched strictly by extract_repository. Do not import extra
|
||||
# tags before introducing() checks the candidate's version bracket.
|
||||
git.run("fetch", "--quiet", "--no-tags", "origin", fix)
|
||||
resolved.append(git.run("rev-parse", fix).strip())
|
||||
except InstrumentFailure as exc:
|
||||
if fix == candidate.fix or not str(exc).startswith("git_exit_"):
|
||||
raise
|
||||
proof = prove_source(git, candidate.parent)
|
||||
change = extract.introducing(git, candidate)
|
||||
known = set()
|
||||
expanded = set()
|
||||
for full in resolved:
|
||||
expanded.add(full)
|
||||
for parent in git.parents(full):
|
||||
sites = extract.fix_sites(git, full, parent, global_ok=True)
|
||||
known.update((site["path"], site["function"]) for site in sites)
|
||||
rows = extract.ordinary_history(git, "HEAD")
|
||||
ordinary, exclusions = extract.draw_ordinary(rows, expanded, known, seed=seed)
|
||||
return [change, *ordinary], exclusions, proof
|
||||
|
||||
|
||||
def inputs(used, seed, known_empty, clones, *, pool_size=300, slices=3):
|
||||
from openultrasast.model.taxonomy import DEFAULT_FAMILIES_PATH
|
||||
|
||||
parameters = {
|
||||
"seed": seed,
|
||||
"used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}),
|
||||
"known_empty": sorted(known_empty),
|
||||
"clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024),
|
||||
"journal_version": 4,
|
||||
"pool_size": pool_size,
|
||||
"slices": slices,
|
||||
}
|
||||
# Hardcoded design thresholds and algorithms are covered as well as taxonomy data.
|
||||
for name in ("draft", "source", "extract", "eligibility", "journal"):
|
||||
parameters["design_" + name] = hashlib.sha256(Path(__file__).with_name(name + ".py").read_bytes()).hexdigest()
|
||||
parameters["design_taxonomy"] = hashlib.sha256(DEFAULT_FAMILIES_PATH.read_bytes()).hexdigest()
|
||||
return parameters
|
||||
|
||||
|
||||
def enumerate_advisories(github, journal):
|
||||
records = {}
|
||||
for ecosystem in source.ECOSYSTEMS:
|
||||
page = 1
|
||||
next_url = None
|
||||
while True:
|
||||
key = f"{ecosystem}:{page}"
|
||||
if key not in journal.rows["advisories"]:
|
||||
journal.boundary()
|
||||
result = github.advisory_page(next_url, ecosystem=ecosystem)
|
||||
journal.put(
|
||||
"advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "next_url": result["next_url"]}
|
||||
)
|
||||
result = journal.rows["advisories"][key]
|
||||
records.update((row["ghsa_id"], row) for row in result["rows"])
|
||||
next_url = result["next_url"]
|
||||
if next_url is None:
|
||||
break
|
||||
page += 1
|
||||
if not records:
|
||||
raise InstrumentFailure("empty_advisories")
|
||||
return list(records.values())
|
||||
|
||||
|
||||
def decide(row, github, osv, used, selected_networks=()):
|
||||
# selected_networks is retained for callers; only ordered acceptance consults it.
|
||||
network = {}
|
||||
try:
|
||||
links = source.fix_links(row)
|
||||
if len(links) != 1:
|
||||
raise source.Rejected("fix_links")
|
||||
name, _ = next(iter(links))
|
||||
resolved = resolve_repository(name, github)
|
||||
rejected_by = eligibility.rejection(resolved, used, set())
|
||||
if rejected_by:
|
||||
raise source.Rejected("used_" + rejected_by)
|
||||
network = {"network": resolved.network}
|
||||
candidate = source.candidate(row, github, osv)
|
||||
return {"candidate": asdict(candidate), "network": resolved.network}
|
||||
except source.Rejected as exc:
|
||||
return {"rejected": str(exc), **network}
|
||||
except eligibility.RepositoryNotFound:
|
||||
return {"rejected": "repository_or_commit_not_found", **network}
|
||||
except ValueError as exc:
|
||||
# A per-candidate transport failure (e.g. get_commit on an unresolvable fix ref)
|
||||
# skips this advisory; the run loop's systemic guard catches a broken instrument.
|
||||
if str(exc) == "http_failure":
|
||||
return {"rejected": "decide_http_failure", **network}
|
||||
raise
|
||||
|
||||
|
||||
def extract_repository(candidate, github, clones, fixes, seed):
|
||||
before = dict(clones.counts)
|
||||
deadline = getattr(clones, "deadline_seconds", None)
|
||||
total = (deadline or 0) + (getattr(clones, "parse_deadline_seconds", None) or 0) if deadline is not None else None
|
||||
try:
|
||||
with extract.process_deadline(total):
|
||||
with clones.open(candidate.url) as git:
|
||||
git.run("fetch", "--quiet", "origin", candidate.fix)
|
||||
changes, exclusions, proof = repository_changes(git, candidate, fixes, seed=seed)
|
||||
entry = {**asdict(candidate), "changes": changes, "instrument": proof}
|
||||
result = {"entry": entry, "ordinary_exclusions": exclusions}
|
||||
except source.Rejected as exc:
|
||||
result = {"rejected": str(exc), "ordinary_exclusions": getattr(exc, "counts", {})}
|
||||
except eligibility.RepositoryNotFound:
|
||||
result = {"rejected": "repository_or_commit_not_found", "ordinary_exclusions": {}}
|
||||
except (extract.InstrumentFailure, OSError) as exc:
|
||||
result = {"rejected": "extract_" + str(exc), "ordinary_exclusions": {}}
|
||||
result["counts"] = {key: value - before.get(key, 0) for key, value in clones.counts.items()}
|
||||
return finish_license(result, github) if github is not None else result
|
||||
|
||||
|
||||
def finish_license(result, github):
|
||||
"""REST stays on the coordinator, including the license at the extracted pin."""
|
||||
if "entry" not in result:
|
||||
return result
|
||||
try:
|
||||
entry = result["entry"]
|
||||
pinned = entry["changes"][0]["head"]
|
||||
record = github.get_license(entry["repository"], pinned)
|
||||
spdx, license_class = source.license_info(record)
|
||||
entry.update(license_spdx=spdx, license_class=license_class, license_ref=pinned, license_path=record.get("path", ""))
|
||||
except (source.Rejected, eligibility.RepositoryNotFound) as exc:
|
||||
reason = "repository_or_commit_not_found" if isinstance(exc, eligibility.RepositoryNotFound) else str(exc)
|
||||
return {"rejected": reason, "ordinary_exclusions": {}, "counts": result["counts"]}
|
||||
except (extract.InstrumentFailure, OSError) as exc:
|
||||
return {"rejected": "extract_" + str(exc), "ordinary_exclusions": {}, "counts": result["counts"]}
|
||||
except ValueError as exc:
|
||||
if str(exc) != "http_failure":
|
||||
raise
|
||||
return {"rejected": "extract_http_failure", "ordinary_exclusions": {}, "counts": result["counts"]}
|
||||
return result
|
||||
|
||||
|
||||
@contextmanager
|
||||
def ordered_extractions(
|
||||
rows,
|
||||
api,
|
||||
osv,
|
||||
used,
|
||||
clones,
|
||||
fixes,
|
||||
seed,
|
||||
checkpoint,
|
||||
entries,
|
||||
workers,
|
||||
pool_size,
|
||||
bank: CorpusBank | None = None,
|
||||
decided=None,
|
||||
stale_resolutions=(),
|
||||
):
|
||||
"""Bounded in-flight work, completion journaling, and seed-order acceptance.
|
||||
|
||||
Only running jobs count against `workers`; completed results wait in the
|
||||
journal while the acceptance cursor is blocked. Counters are private to each
|
||||
job, including clone cleanup, and merged exactly once on completion here.
|
||||
Graceful STOP/guard/target exits drain started work into the journal for resume;
|
||||
a hard interrupt may lose unjournaled work but cannot change accepted order.
|
||||
"""
|
||||
decided = dict(decided or {})
|
||||
identities = {}
|
||||
pending = deque() # Seed-order acceptance cursor; contains no futures/results.
|
||||
in_flight = {} # Future -> extraction key, independent of the cursor.
|
||||
# Fork shares the coordinator's memory copy-on-write. Jobs receive only the
|
||||
# picklable extraction inputs, never the journal, REST clients or stores.
|
||||
context = multiprocessing.get_context("fork")
|
||||
pool = ProcessPoolExecutor(max_workers=workers, mp_context=context)
|
||||
|
||||
def resolution(row):
|
||||
key = row["ghsa_id"]
|
||||
if key not in checkpoint.rows["eligibility"]:
|
||||
result = decide(row, api, osv, used)
|
||||
if bank is not None:
|
||||
bank.bank_resolution(key, result, contract=RESOLUTION_CONTRACT)
|
||||
checkpoint.put("eligibility", key, result)
|
||||
return checkpoint.rows["eligibility"][key]
|
||||
|
||||
def remember(repository, fix, result):
|
||||
bank.bank(repository, fix, result, contract=EXTRACTION_CONTRACT)
|
||||
# Reuse decisions made during this invocation too: another advisory for
|
||||
# the same pair must not overwrite an earlier accept with a rejection.
|
||||
decided[bank.key_for(repository, fix)] = {
|
||||
"repository": repository,
|
||||
"fix": fix,
|
||||
"decision": "accept" if "entry" in result else "reject",
|
||||
"entry": result.get("entry"),
|
||||
"reason": result.get("rejected"),
|
||||
"ordinary_exclusions": result.get("ordinary_exclusions", {}),
|
||||
"counts": result.get("counts", {}),
|
||||
}
|
||||
|
||||
def collect(future):
|
||||
key = in_flight.pop(future)
|
||||
try:
|
||||
result = future.result()
|
||||
except BrokenProcessPool:
|
||||
# A killed worker also fails unrelated outstanding futures. Do not
|
||||
# mistake these collateral failures for a broken git instrument.
|
||||
result = {"rejected": "worker_crash", "ordinary_exclusions": {}, "counts": {}}
|
||||
except Exception:
|
||||
# Includes task/result serialization failures. Keep diagnostics
|
||||
# identity-free; repeated task errors still reach the normal guard.
|
||||
result = {"rejected": "extract_worker_failure", "ordinary_exclusions": {}, "counts": {}}
|
||||
for name, value in result["counts"].items():
|
||||
if name == "peak_clone_bytes":
|
||||
clones.counts[name] = max(clones.counts[name], value)
|
||||
else:
|
||||
clones.counts[name] += value
|
||||
result = finish_license(result, api)
|
||||
if bank is not None:
|
||||
repository, fix = identities[key]
|
||||
remember(repository, fix, result)
|
||||
checkpoint.put("extraction", key, result)
|
||||
|
||||
def iterate():
|
||||
nonlocal pool
|
||||
ordered = iter(rows)
|
||||
exhausted = False
|
||||
|
||||
def ready():
|
||||
return pending and (pending[0][2] is None or pending[0][2] in checkpoint.rows["extraction"])
|
||||
|
||||
while (pending or not exhausted) and len(entries) < pool_size:
|
||||
# Only this cursor feeds acceptance, fork deduplication and the guard.
|
||||
# Replayed results follow the same path and need no worker submission.
|
||||
while ready():
|
||||
checkpoint.boundary()
|
||||
row, decision, key = pending.popleft()
|
||||
yield row, decision, checkpoint.rows["extraction"].get(key)
|
||||
if len(entries) == pool_size:
|
||||
return
|
||||
while not exhausted and len(in_flight) < min(workers, pool_size - len(entries)):
|
||||
checkpoint.boundary()
|
||||
row = next(ordered, None)
|
||||
if row is None:
|
||||
exhausted = True
|
||||
break
|
||||
key = row["ghsa_id"]
|
||||
if key in stale_resolutions:
|
||||
resolution(row)
|
||||
identity = None
|
||||
if bank is not None:
|
||||
links = source.fix_links(row)
|
||||
if len(links) == 1:
|
||||
repository, fix = next(iter(links))
|
||||
try:
|
||||
resolved = resolve_repository(repository, api)
|
||||
repository = resolved.full_name
|
||||
except eligibility.RepositoryNotFound:
|
||||
resolved = None
|
||||
except ValueError as exc:
|
||||
if str(exc) != "http_failure":
|
||||
raise
|
||||
resolved = None # decide() records this as a resilient skip.
|
||||
identity = (repository, fix)
|
||||
saved = decided.get(bank.key_for(*identity))
|
||||
if saved is None and len(fix) < 40:
|
||||
matches = [
|
||||
value for value in decided.values() if value["repository"] == repository and value["fix"].startswith(fix)
|
||||
]
|
||||
if len(matches) == 1:
|
||||
saved = matches[0]
|
||||
identity = (repository, saved["fix"])
|
||||
if saved is None:
|
||||
# A duplicate advisory can arrive before its pair's
|
||||
# worker finishes. Share that pending result as well.
|
||||
matching = [key for key, pair in identities.items() if pair[0] == repository and pair[1].startswith(fix)]
|
||||
if len(matching) == 1:
|
||||
pending.append((row, {"network": resolved.network if resolved else repository}, matching[0]))
|
||||
break
|
||||
if saved is not None:
|
||||
decision = {"network": resolved.network if resolved is not None else repository}
|
||||
blocked = eligibility.rejection(resolved, used, set()) if resolved is not None else None
|
||||
if blocked:
|
||||
decision["rejected"] = "used_" + blocked
|
||||
pending.append((row, decision, None))
|
||||
else:
|
||||
result = {"ordinary_exclusions": saved["ordinary_exclusions"], "counts": saved["counts"]}
|
||||
if saved["decision"] == "accept":
|
||||
result["entry"] = saved["entry"]
|
||||
else:
|
||||
result["rejected"] = saved["reason"]
|
||||
extraction_key = bank.key_for(*identity)
|
||||
if extraction_key not in checkpoint.rows["extraction"]:
|
||||
checkpoint.put("extraction", extraction_key, result)
|
||||
pending.append((row, decision, extraction_key))
|
||||
break
|
||||
decision = resolution(row)
|
||||
extraction_key, future = None, None
|
||||
if "candidate" in decision:
|
||||
candidate = source.Candidate(**decision["candidate"])
|
||||
extraction_key = digest({"repository": candidate.repository, "advisory": key, "fix": candidate.fix})
|
||||
if bank is not None:
|
||||
extraction_key = bank.key_for(candidate.repository, candidate.fix)
|
||||
identities[extraction_key] = (candidate.repository, candidate.fix)
|
||||
if extraction_key not in checkpoint.rows["extraction"] and extraction_key not in in_flight.values():
|
||||
checkpoint.boundary()
|
||||
local = copy.copy(clones)
|
||||
local.counts = Counter()
|
||||
args = (candidate, None, local, fixes.get(candidate.repository, set()), seed)
|
||||
try:
|
||||
future = pool.submit(extract_repository, *args)
|
||||
except BrokenProcessPool:
|
||||
# Already-submitted jobs retain their ordered crash
|
||||
# skips; only new jobs enter the replacement pool.
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
pool = ProcessPoolExecutor(max_workers=workers, mp_context=context)
|
||||
future = pool.submit(extract_repository, *args)
|
||||
in_flight[future] = extraction_key
|
||||
elif bank is not None and identity is not None:
|
||||
remember(*identity, decision)
|
||||
pending.append((row, decision, extraction_key))
|
||||
# Finalize cached decisions promptly (especially on resume), and
|
||||
# service completed jobs even during a long run of decide skips.
|
||||
if ready() or any(future.done() for future in in_flight):
|
||||
break
|
||||
if ready():
|
||||
continue
|
||||
if in_flight:
|
||||
checkpoint.boundary()
|
||||
done, _ = wait(in_flight, return_when=FIRST_COMPLETED)
|
||||
for future in done:
|
||||
collect(future)
|
||||
# Freed slots are refilled on the next pass even if the oldest
|
||||
# candidate is still running. No result() waits on that cursor.
|
||||
|
||||
interrupted = False
|
||||
stopped = None
|
||||
try:
|
||||
yield iterate()
|
||||
except BaseException as exc:
|
||||
interrupted = not isinstance(exc, Exception)
|
||||
stopped = str(exc) if isinstance(exc, Stopped) else None
|
||||
raise
|
||||
finally:
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
if not interrupted:
|
||||
for future in list(in_flight):
|
||||
if not future.cancelled():
|
||||
collect(future)
|
||||
if stopped is not None:
|
||||
checkpoint.progress("stopped", reason=stopped)
|
||||
|
||||
|
||||
def publish(journal, assigned, seed, *, destination=None):
|
||||
"""Idempotent publication from completed journal data, never inside the checkout."""
|
||||
public, private = [], []
|
||||
for row in assigned:
|
||||
if row["license_class"] == "permissive":
|
||||
public.append(row)
|
||||
else:
|
||||
private.append(row)
|
||||
public.append({"id": row["id"], "sha256": private_digest(row), "slice": row["slice"], "private": True})
|
||||
for name, rows in (("private-draft-p1.toml", private), ("draft-p1.toml", public)):
|
||||
path = (destination or journal.path) / name
|
||||
data = manifest(rows, seed).encode()
|
||||
if path.exists():
|
||||
if path.read_bytes() != data:
|
||||
raise ValueError("draft_exists")
|
||||
else:
|
||||
atomic(path, data)
|
||||
|
||||
|
||||
def run(
|
||||
root: Path,
|
||||
*,
|
||||
github,
|
||||
osv,
|
||||
clones,
|
||||
used: eligibility.UsedSet,
|
||||
seed: int,
|
||||
known_empty=None,
|
||||
journal_path=None,
|
||||
max_hours=None,
|
||||
clock=None,
|
||||
extract_workers=None,
|
||||
pool_size=300,
|
||||
slices=3,
|
||||
bank: CorpusBank | None = None,
|
||||
) -> dict:
|
||||
validate_pool(pool_size, slices)
|
||||
workers = extract_workers if extract_workers is not None else getattr(clones, "extract_workers", 6)
|
||||
if workers < 1:
|
||||
raise ValueError("invalid_extract_workers")
|
||||
workers = min(workers, os.cpu_count() or 4)
|
||||
started = time.monotonic()
|
||||
report = {"status": "sourcing", "seed": seed, "candidates": {}, "rejections": {}, "ordinary_exclusions": {}}
|
||||
# Operational budget only: changing it must not invalidate journal resume inputs.
|
||||
report["extract_workers"] = workers
|
||||
report["extract_deadline_seconds"] = getattr(clones, "deadline_seconds", None)
|
||||
report["parse_deadline_seconds"] = getattr(clones, "parse_deadline_seconds", None)
|
||||
counts, rejected, ordinary_counts = Counter(), Counter(), Counter()
|
||||
checkpoint = None
|
||||
scratch = ExitStack()
|
||||
try:
|
||||
decided = bank.decided() if bank is not None else {}
|
||||
banked = bank.resolved() if bank is not None else {}
|
||||
accepted, reusable = split_decided(decided, contract=EXTRACTION_CONTRACT)
|
||||
decided = {key: value for key, value in decided.items() if key in reusable}
|
||||
known_empty = set(known_empty or ())
|
||||
used.report(known_empty)
|
||||
cache = getattr(clones, "cache", root.parent / (root.name + "-cache"))
|
||||
path = external(root, journal_path or cache / "journal")
|
||||
journal_directory = path
|
||||
if bank is not None:
|
||||
# Local records are a disposable cursor, never a cross-run authority.
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
journal_directory = Path(scratch.enter_context(tempfile.TemporaryDirectory(prefix="cursor-", dir=path)))
|
||||
checkpoint = Journal(
|
||||
journal_directory,
|
||||
max_hours=max_hours,
|
||||
clock=clock,
|
||||
rate_limit=lambda: getattr(getattr(github, "transport", None), "rate_limit", {}),
|
||||
)
|
||||
with checkpoint.open(inputs(used, seed, known_empty, clones, pool_size=pool_size, slices=slices)):
|
||||
checkpoint.boundary()
|
||||
api = BankedGitHub(github, checkpoint, bank) if bank is not None else ResolvedGitHub(github, checkpoint)
|
||||
checkpoint.rows["eligibility"].update(
|
||||
{key: body["result"] for key, body in banked.items() if body["resolution_contract"] == RESOLUTION_CONTRACT}
|
||||
)
|
||||
used = copy.deepcopy(used) # canonicalization must not mutate the resume input
|
||||
eligibility.canonicalize_used(used, api)
|
||||
report["used_set"] = used.report(known_empty)
|
||||
rows = bank.cached_advisories(contract=RESOLUTION_CONTRACT) if bank is not None else None
|
||||
if rows is None:
|
||||
try:
|
||||
rows = enumerate_advisories(api, checkpoint)
|
||||
except InstrumentFailure as exc:
|
||||
if str(exc) != "empty_advisories" or not accepted:
|
||||
raise
|
||||
rows = []
|
||||
if bank is not None:
|
||||
bank.cache_advisories(rows, contract=RESOLUTION_CONTRACT)
|
||||
if bank is not None:
|
||||
# Retain accepted entries even if their advisory disappeared upstream.
|
||||
present = {row["ghsa_id"] for row in rows}
|
||||
for entry in accepted:
|
||||
if entry["advisory"] not in present:
|
||||
rows.append(
|
||||
{
|
||||
"ghsa_id": entry["advisory"],
|
||||
"published_at": "2026-06-02" if entry["post_cutoff"] else "2026-06-01",
|
||||
"vulnerabilities": [{"package": {"ecosystem": entry["ecosystem"]}}],
|
||||
"references": [f"https://github.com/{entry['repository']}/commit/{entry['fix']}"],
|
||||
}
|
||||
)
|
||||
present.add(entry["advisory"])
|
||||
report["advisories"] = len(rows)
|
||||
fixes_by_repo = advisory_index(rows, api)
|
||||
selected_networks, entries = set(), []
|
||||
instrument_failures = 0
|
||||
with ordered_extractions(
|
||||
source.ordered_advisories(rows, seed),
|
||||
api,
|
||||
osv,
|
||||
used,
|
||||
clones,
|
||||
fixes_by_repo,
|
||||
seed,
|
||||
checkpoint,
|
||||
entries,
|
||||
workers,
|
||||
pool_size,
|
||||
bank=bank,
|
||||
decided=decided,
|
||||
stale_resolutions={key for key, body in banked.items() if body["resolution_contract"] != RESOLUTION_CONTRACT},
|
||||
) as outcomes:
|
||||
for row, decision, result in outcomes:
|
||||
counts[source.ecosystem(row)] += 1
|
||||
if decision.get("network") in selected_networks:
|
||||
# Serial decide skipped this candidate entirely. Even a failed
|
||||
# speculative extraction must not affect the systemic guard.
|
||||
rejected["fork_network"] += 1
|
||||
elif "rejected" in decision:
|
||||
rejected[decision["rejected"]] += 1
|
||||
instrument_failures += int(decision["rejected"] == "decide_http_failure")
|
||||
else:
|
||||
ordinary_counts.update(result["ordinary_exclusions"])
|
||||
if "rejected" in result:
|
||||
rejected[result["rejected"]] += 1
|
||||
instrument_failures += int(
|
||||
result["rejected"].startswith("extract_") or result["rejected"] == "decide_http_failure"
|
||||
)
|
||||
else:
|
||||
selected_networks.add(decision["network"])
|
||||
entries.append(result["entry"])
|
||||
# A low-yield draw can accumulate unbounded candidate failures;
|
||||
# only zero accepted entries indicate a systemic failure.
|
||||
if instrument_failures >= 25 and not entries:
|
||||
raise extract.InstrumentFailure("systemic_extraction_failure")
|
||||
if len(entries) == pool_size:
|
||||
break
|
||||
report.update(
|
||||
{
|
||||
"repositories": len(entries),
|
||||
"candidates": dict(counts),
|
||||
"rejections": dict(rejected),
|
||||
"ordinary_exclusions": dict(ordinary_counts),
|
||||
}
|
||||
)
|
||||
checkpoint.boundary()
|
||||
if len(entries) != pool_size:
|
||||
report["status"] = "insufficient_repositories"
|
||||
checkpoint.progress(report["status"])
|
||||
return report
|
||||
assigned = assign_slices(entries, seed=seed, pool_size=pool_size, slices=slices)
|
||||
completion = {"manifest_sha256": digest(assigned), "repositories": len(entries)}
|
||||
if "all" not in checkpoint.rows["completion"]:
|
||||
checkpoint.put("completion", "all", completion)
|
||||
elif checkpoint.rows["completion"]["all"] != completion:
|
||||
raise ValueError("journal_completion_mismatch")
|
||||
checkpoint.boundary()
|
||||
publish(checkpoint, assigned, seed, destination=path if bank is not None else None)
|
||||
report.update(
|
||||
{
|
||||
"status": "draft",
|
||||
"changes": pool_size * 13,
|
||||
"slices": slices,
|
||||
"license_split": dict(Counter(e["license_class"] for e in entries)),
|
||||
"methods": dict(Counter(e["changes"][0]["method"] for e in entries)),
|
||||
"post_cutoff": sum(e["post_cutoff"] for e in entries),
|
||||
**completion,
|
||||
}
|
||||
)
|
||||
checkpoint.progress("draft")
|
||||
return report
|
||||
except ResumeMismatch as exc:
|
||||
report.update({"status": "resume_refused", "changed_parameters": exc.parameters})
|
||||
return report
|
||||
except Stopped as exc:
|
||||
report.update({"status": "stopped", "reason": str(exc)})
|
||||
return report
|
||||
except Exception as exc:
|
||||
report.update({"status": "instrument_failure", "error_type": type(exc).__name__})
|
||||
return report
|
||||
finally:
|
||||
scratch.close()
|
||||
report.update({"candidates": dict(counts), "rejections": dict(rejected), "ordinary_exclusions": dict(ordinary_counts)})
|
||||
report["instrument"] = dict(clones.counts)
|
||||
report["wall_seconds"] = round(time.monotonic() - started, 3)
|
||||
report["github_calls"] = github.calls
|
||||
report["osv_calls"] = osv.calls
|
||||
report["github_http_attempts"] = getattr(getattr(github, "transport", None), "attempts", github.calls)
|
||||
report["osv_http_attempts"] = getattr(getattr(osv, "transport", None), "attempts", osv.calls)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
from openultrasast import config
|
||||
from openultrasast.plane import memory
|
||||
from openultrasast.plane.reconciler import results_root
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--cache", type=Path)
|
||||
parser.add_argument("--max-hours", type=float, help="Stop between units after this many hours per invocation")
|
||||
parser.add_argument("--extract-deadline", type=float, default=240.0, help="Extraction wall-clock budget per repository in seconds")
|
||||
parser.add_argument(
|
||||
"--parse-deadline",
|
||||
type=float,
|
||||
default=120.0,
|
||||
help="Extra wall-clock budget in seconds for the Python parse phase beyond --extract-deadline",
|
||||
)
|
||||
parser.add_argument("--extract-workers", type=int, default=6, help="Maximum extraction processes, capped to CPU cores (default: 6)")
|
||||
parser.add_argument("--pool-size", type=int, default=300, help="Repositories to accept (default: 300)")
|
||||
parser.add_argument("--slices", type=int, default=3, help="Equal-sized slices (default: 3)")
|
||||
parser.add_argument("--seed", type=int, default=20260601)
|
||||
parser.add_argument("--bank", nargs="?", const="unseen", default="", help="Corpus bucket prefix; enables durable decisions")
|
||||
parser.add_argument("--local-store")
|
||||
parser.add_argument("--s3-store", default="s3://")
|
||||
parser.add_argument("--pointer-manifest", type=Path, action="append", default=[])
|
||||
parser.add_argument("--known-empty", choices=(*eligibility.SOURCES, "memory_local", "memory_s3"), action="append", default=[])
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
validate_pool(args.pool_size, args.slices)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
if args.extract_workers < 1:
|
||||
parser.error("--extract-workers must be at least 1")
|
||||
try:
|
||||
config.load_dotenv(args.root / ".env")
|
||||
# Refuse an existing draft before doing any network work.
|
||||
if any((args.root / p).exists() for p in ("benchmarks/unseen/draft-p1.toml", "benchmarks/unseen/private/draft-p1.toml")):
|
||||
raise FileExistsError("draft_exists")
|
||||
clones = Clones(
|
||||
args.root,
|
||||
args.cache or results_root() / "unseen-draft",
|
||||
deadline_seconds=args.extract_deadline,
|
||||
parse_deadline_seconds=args.parse_deadline,
|
||||
extract_workers=args.extract_workers,
|
||||
)
|
||||
stores = {
|
||||
"local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True),
|
||||
"s3": memory.open_store(args.s3_store, read_only=True),
|
||||
}
|
||||
used = eligibility.build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest)
|
||||
bank = CorpusBank(CorpusObjects(**corpus_settings()), seed=args.seed, prefix=args.bank) if args.bank else None
|
||||
report = run(
|
||||
args.root,
|
||||
github=source.GitHub(os.environ.get("GH_TOKEN", "")),
|
||||
osv=source.OSV(),
|
||||
clones=clones,
|
||||
used=used,
|
||||
seed=args.seed,
|
||||
pool_size=args.pool_size,
|
||||
slices=args.slices,
|
||||
known_empty=set(args.known_empty),
|
||||
journal_path=clones.cache / "journal",
|
||||
max_hours=args.max_hours,
|
||||
bank=bank,
|
||||
)
|
||||
except Exception as exc:
|
||||
report = {"status": "instrument_failure", "error_type": type(exc).__name__}
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return 0 if report["status"] in {"draft", "stopped"} else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,379 +0,0 @@
|
||||
"""Read-only used-set collection. Only counts, source labels and digests leave this module's CLI.
|
||||
|
||||
Run from the repository root: python -m benchmarks.unseen.eligibility --help.
|
||||
Population v3 is exclusively the existing guard's responsibility.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import tomllib
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
from openultrasast.plane.memory import MemoryStore
|
||||
|
||||
URL = re.compile(r"github\.com/([\w.-]+/[\w.-]+)", re.I)
|
||||
NAME = re.compile(r"[\w.-]+/[\w.-]+")
|
||||
SOURCES = ("pairs", "pointers", "populations", "harvest", "units", "measurements", "sweep")
|
||||
|
||||
|
||||
def normalize(value: str) -> str:
|
||||
text = value.strip().lower().rstrip("/")
|
||||
if NAME.fullmatch(text):
|
||||
name = text.removesuffix(".git")
|
||||
else:
|
||||
# Structured fields must be identities, not prose containing a URL.
|
||||
if any(char.isspace() for char in text):
|
||||
raise ValueError("invalid_repository")
|
||||
url = urlsplit(text if "://" in text else "https://" + text)
|
||||
if url.scheme not in {"http", "https", "ssh", "git"} or not url.hostname or "." not in url.hostname:
|
||||
raise ValueError("invalid_repository")
|
||||
parts = url.path.strip("/").split("/")
|
||||
if len(parts) < 2:
|
||||
raise ValueError("invalid_repository")
|
||||
name = "/".join(parts[:2]).removesuffix(".git")
|
||||
if not NAME.fullmatch(name):
|
||||
raise ValueError("invalid_repository")
|
||||
if url.hostname != "github.com":
|
||||
return url.hostname + "/" + name
|
||||
if not NAME.fullmatch(name) or any(part in {".", ".."} for part in name.split("/")):
|
||||
raise ValueError("invalid_repository")
|
||||
return name
|
||||
|
||||
|
||||
def excluded(path: Path) -> bool:
|
||||
return any("v3" in part.lower() or part in {".git", "__pycache__", ".venv"} for part in path.parts)
|
||||
|
||||
|
||||
def repositories(value: object, counts: dict[str, int] | None = None) -> set[str]:
|
||||
"""Structured repo fields include bare owner/name pointers; URLs also occur in prose."""
|
||||
found = set()
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key in {"repo", "fix_repo", "repository", "full_name", "origin"} or (key == "group" and "unit" in value):
|
||||
try:
|
||||
if not isinstance(item, str):
|
||||
raise ValueError("invalid_repository")
|
||||
found.add(normalize(item))
|
||||
except ValueError:
|
||||
if counts is not None:
|
||||
counts["skipped_values"] = counts.get("skipped_values", 0) + 1
|
||||
else:
|
||||
found.update(repositories(item, counts))
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found.update(repositories(item, counts))
|
||||
elif isinstance(value, str):
|
||||
found.update(normalize(m[1]) for m in URL.finditer(value))
|
||||
return found
|
||||
|
||||
|
||||
def pointer_repositories(value: object, counts: dict[str, int] | None = None) -> set[str]:
|
||||
if isinstance(value, dict):
|
||||
if value.get("vendored") is False:
|
||||
return repositories(value, counts)
|
||||
return set().union(*(pointer_repositories(v, counts) for v in value.values()))
|
||||
if isinstance(value, list):
|
||||
return set().union(*(pointer_repositories(v, counts) for v in value))
|
||||
return set()
|
||||
|
||||
|
||||
def parse(path: Path, raw: bytes) -> object:
|
||||
structured = path.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"}
|
||||
text = raw.decode("utf-8", errors="strict" if structured else "ignore")
|
||||
if path.suffix == ".toml":
|
||||
return tomllib.loads(text)
|
||||
if path.suffix == ".json":
|
||||
return json.loads(text)
|
||||
if path.suffix == ".jsonl":
|
||||
return [json.loads(line) for line in text.splitlines() if line.strip()]
|
||||
if path.suffix in {".yaml", ".yml"}:
|
||||
documents = list(yaml.safe_load_all(text))
|
||||
return documents[0] if len(documents) == 1 else documents
|
||||
return text
|
||||
|
||||
|
||||
@dataclass
|
||||
class UsedSet:
|
||||
sources: dict[str, set[str]] = field(default_factory=lambda: {label: set() for label in SOURCES})
|
||||
instrument: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
|
||||
def add(self, label: str, names: set[str], size: int, files: int = 1, skipped_values: int = 0) -> None:
|
||||
self.sources.setdefault(label, set()).update(names)
|
||||
counts = self.instrument.setdefault(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0})
|
||||
counts["files"] += files
|
||||
counts["bytes"] += size
|
||||
counts["rows"] += len(names)
|
||||
counts["skipped_values"] += skipped_values
|
||||
|
||||
def report(self, known_empty: set[str] | None = None) -> dict:
|
||||
known_empty = known_empty or set()
|
||||
if known_empty - self.sources.keys():
|
||||
raise ValueError("unknown_source_label")
|
||||
records = {}
|
||||
for label, names in self.sources.items():
|
||||
counts = self.instrument.get(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0})
|
||||
if not counts["rows"] and label not in known_empty:
|
||||
raise ValueError("empty_source:" + label)
|
||||
records[label] = {**counts, "count": len(names), "known_empty": label in known_empty}
|
||||
names = set().union(*self.sources.values())
|
||||
return {"sources": records, "count": len(names), "sha256": hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest()}
|
||||
|
||||
|
||||
def build_used(root: Path, *, stores: Mapping[str, MemoryStore] | None = None, pointer_manifests: Iterable[Path] = ()) -> UsedSet:
|
||||
used = UsedSet()
|
||||
|
||||
def read(label: str, paths: Iterable[Path], structured: bool = True) -> list[object]:
|
||||
values = []
|
||||
for path in sorted(set(paths)):
|
||||
if excluded(path):
|
||||
continue
|
||||
raw = path.read_bytes() # failure is an instrument failure, never a quiet empty source
|
||||
value = parse(path, raw) if structured else raw.decode("utf-8", errors="ignore")
|
||||
counts: dict[str, int] = {}
|
||||
names = repositories(value, counts)
|
||||
used.add(label, names, len(raw), skipped_values=counts.get("skipped_values", 0))
|
||||
if label == "pairs":
|
||||
pointer_counts: dict[str, int] = {}
|
||||
pointers = pointer_repositories(value, pointer_counts)
|
||||
if pointers or pointer_counts:
|
||||
used.add("pointers", pointers, len(raw), skipped_values=pointer_counts.get("skipped_values", 0))
|
||||
values.append(value)
|
||||
return values
|
||||
|
||||
pairs = root / "benchmarks/pairs"
|
||||
read("pairs", (p for p in pairs.rglob("*") if p.is_file() and p.suffix in {".toml", ".json", ".jsonl", ".csv", ".yaml", ".yml"}))
|
||||
read("pointers", pointer_manifests)
|
||||
read("populations", (p for p in (root / "benchmarks/independent").glob("population-v[12].toml")))
|
||||
read("harvest", (p for directory in pairs.glob("advisory-fixes*") for p in directory.rglob("*.toml")))
|
||||
units = set((root / "plane/experiments").glob("*.units.jsonl"))
|
||||
for path in (root / "plane/experiments").glob("*.y*ml"):
|
||||
if excluded(path):
|
||||
continue
|
||||
raw = path.read_bytes()
|
||||
manifest = parse(path, raw)
|
||||
counts = {}
|
||||
names = repositories(manifest, counts)
|
||||
used.add("units", names, len(raw), skipped_values=counts.get("skipped_values", 0))
|
||||
if isinstance(manifest, dict) and manifest.get("units_file"):
|
||||
declared = root / Path(manifest["units_file"]).expanduser()
|
||||
if declared.exists(): # an unregistered plan may not have frozen units yet
|
||||
units.add(declared)
|
||||
for label, store in (stores or {}).items():
|
||||
if label not in {"local", "s3"}:
|
||||
raise ValueError("unknown_store_label")
|
||||
source = "memory_" + label
|
||||
used.add(source, set(), 0, 0)
|
||||
# Read actual objects, not a serialized row size masquerading as bytes read.
|
||||
for key in store._keys("repos/"):
|
||||
if not key.endswith(".jsonl") or excluded(Path(key)):
|
||||
continue
|
||||
result = store._get(key)
|
||||
if result is None:
|
||||
raise ValueError("missing_memory_object")
|
||||
raw = result[0]
|
||||
rows = parse(Path("rows.jsonl"), raw)
|
||||
names = set()
|
||||
counts = {}
|
||||
for row in rows:
|
||||
if row.get("kind") == "example" and "repo" in row:
|
||||
names.update(repositories({"repo": row["repo"]}, counts))
|
||||
if row.get("kind") == "experiment" and row.get("units_file"):
|
||||
registered = root / Path(row["units_file"]).expanduser()
|
||||
if row.get("units_digest") or registered.exists():
|
||||
units.add(registered)
|
||||
used.add(source, names, len(raw), skipped_values=counts.get("skipped_values", 0))
|
||||
read("units", units)
|
||||
read("measurements", (p for top in ("measurements", "experiments") for p in (root / "benchmarks" / top).rglob("*") if p.is_file()))
|
||||
read(
|
||||
"sweep",
|
||||
(
|
||||
p
|
||||
for top in ("benchmarks", "src", "tests", "plane", ".kiro")
|
||||
for p in (root / top).rglob("*")
|
||||
if p.is_file() and not p.is_relative_to(root / "benchmarks/unseen") and not excluded(p)
|
||||
),
|
||||
structured=False,
|
||||
)
|
||||
return used
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Resolution:
|
||||
full_name: str
|
||||
names: frozenset[str]
|
||||
network: str
|
||||
|
||||
|
||||
def resolve(candidate: str, api) -> Resolution:
|
||||
name = normalize(candidate)
|
||||
if name.count("/") > 1:
|
||||
return Resolution(name, frozenset({name}), name)
|
||||
data = api.get_repo(name)
|
||||
canonical = normalize(data["full_name"])
|
||||
names = {name, canonical, *(normalize(n) for n in data.get("former_names", []))}
|
||||
for key in ("parent", "source"):
|
||||
if data.get(key):
|
||||
names.add(normalize(data[key]["full_name"]))
|
||||
network = normalize((data.get("source") or data.get("parent") or data)["full_name"])
|
||||
return Resolution(canonical, frozenset(names), network)
|
||||
|
||||
|
||||
def canonicalize_used(used: UsedSet, api) -> None:
|
||||
"""Resolve old used names too: GitHub exposes redirects, not a complete rename history."""
|
||||
cache = {}
|
||||
for names in used.sources.values():
|
||||
for name in sorted(names):
|
||||
if name not in cache:
|
||||
try:
|
||||
cache[name] = resolve(name, api)
|
||||
except RepositoryNotFound:
|
||||
cache[name] = Resolution(name, frozenset({name}), name)
|
||||
names.update(cache[name].names)
|
||||
|
||||
|
||||
def rejection(candidate: Resolution, used: UsedSet, selected: set[str]) -> str | None:
|
||||
for label, names in used.sources.items():
|
||||
if candidate.names & names:
|
||||
return label
|
||||
if candidate.network in selected:
|
||||
return "fork_network"
|
||||
selected.add(candidate.network)
|
||||
return None
|
||||
|
||||
|
||||
class RepositoryNotFound(ValueError):
|
||||
"""A historical or synthetic used identity no longer resolves; retain it verbatim."""
|
||||
|
||||
|
||||
class GitHub:
|
||||
"""Coordinator-only GET client. Exceptions never disclose request URLs or tokens."""
|
||||
|
||||
def __init__(self, token: str):
|
||||
if not token:
|
||||
raise ValueError("missing_gh_token")
|
||||
self.token = token
|
||||
|
||||
def get_repo(self, name: str) -> dict:
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
request = Request(
|
||||
"https://api.github.com/repos/" + normalize(name),
|
||||
headers={
|
||||
"Authorization": "Bearer " + self.token,
|
||||
"Accept": "application/vnd.github+json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=30) as response:
|
||||
return json.load(response)
|
||||
except HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
raise RepositoryNotFound("github_not_found") from None
|
||||
raise ValueError("github_resolution_failed") from None
|
||||
except Exception:
|
||||
raise ValueError("github_resolution_failed") from None
|
||||
|
||||
|
||||
def verify_tests(root: Path) -> dict:
|
||||
"""The coordinator runs the guard as a black box; only exit codes and counts escape."""
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from xml.etree import ElementTree
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
report = Path(directory) / "tests.xml"
|
||||
done = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"-q",
|
||||
"-p",
|
||||
"no:cacheprovider",
|
||||
"tests/test_unseen_eligibility.py",
|
||||
"tests/test_unseen_ledger.py",
|
||||
"tests/test_independent_population.py",
|
||||
"--junitxml=" + str(report),
|
||||
],
|
||||
cwd=root,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
if done.returncode != 0:
|
||||
raise ValueError("guard_or_tests_failed")
|
||||
suites = ElementTree.parse(report).getroot().iter("testsuite")
|
||||
counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")}
|
||||
for suite in suites:
|
||||
for key in counts:
|
||||
counts[key] += int(suite.attrib[key])
|
||||
if not counts["tests"]:
|
||||
raise ValueError("empty_test_run")
|
||||
return {"guard": "green", "test_exit_code": done.returncode, "tests": counts}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
import os
|
||||
|
||||
from openultrasast import config
|
||||
from openultrasast.plane import memory
|
||||
from openultrasast.plane.reconciler import results_root
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--local-store", default=None)
|
||||
parser.add_argument("--s3-store", default="s3://")
|
||||
parser.add_argument("--pointer-manifest", type=Path, action="append", default=[])
|
||||
parser.add_argument("--known-empty", choices=(*SOURCES, "memory_local", "memory_s3"), action="append", default=[])
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--verify-tests", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
step = "open_stores"
|
||||
try:
|
||||
config.load_dotenv(args.root / ".env")
|
||||
stores = {
|
||||
"local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True),
|
||||
"s3": memory.open_store(args.s3_store, read_only=True),
|
||||
}
|
||||
step = "build_used"
|
||||
used = build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest)
|
||||
step = "report"
|
||||
used.report(set(args.known_empty)) # prove inputs before API work
|
||||
step = "canonicalize"
|
||||
canonicalize_used(used, GitHub(os.environ.get("GH_TOKEN", "")))
|
||||
step = "report"
|
||||
report = used.report(set(args.known_empty))
|
||||
if args.verify_tests:
|
||||
step = "verify_tests"
|
||||
report.update(verify_tests(args.root))
|
||||
step = "report"
|
||||
text = json.dumps(report, sort_keys=True, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("x") as handle:
|
||||
handle.write(text)
|
||||
print(text, end="")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# Source paths, store exceptions and JSON parser errors can contain repository names.
|
||||
report = {"status": "instrument_failure", "step": step, "type": type(exc).__name__}
|
||||
if str(exc) in {"empty_source:" + label for label in (*SOURCES, "memory_local", "memory_s3")}:
|
||||
report["source"] = str(exc).partition(":")[2]
|
||||
print(json.dumps(report))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,548 +0,0 @@
|
||||
"""Git-only change extraction; clients and subprocess execution are injectable.
|
||||
|
||||
No scanner, engine or model runs here. Function boundaries are syntax, not a security
|
||||
judgement: the first removed code line of a fix is the SZZ sink proxy. Ambiguous sites
|
||||
are dropped, and the recorded SZZ bias remains subject to task 2.6's label check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from .source import Candidate, Rejected
|
||||
|
||||
# Bump only when accept/reject outcomes or accepted entry bytes change.
|
||||
EXTRACTION_CONTRACT = "v1"
|
||||
# Bump only when enumeration, eligibility or source.candidate outputs change.
|
||||
RESOLUTION_CONTRACT = "v1"
|
||||
|
||||
LANGUAGES = {".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java"}
|
||||
HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.M)
|
||||
SHA = re.compile(r"[a-f0-9]{40}")
|
||||
BIAS = {
|
||||
"introducing": "SZZ last-writer proxy: refactors, moves and reformats can misidentify the introduction in either direction.",
|
||||
"last_touch": "Pessimistic lower bound: the flaw may already exist in base, so a delta hook can correctly stay quiet.",
|
||||
}
|
||||
|
||||
|
||||
class InstrumentFailure(ValueError):
|
||||
"""A failed read or command must never masquerade as an empty history."""
|
||||
|
||||
|
||||
class ExtractionTimeout(Rejected):
|
||||
"""A slow repository is skipped without aborting the draft."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def process_deadline(seconds: float | None):
|
||||
"""Bound Python work in a worker's main thread; cancel the alarm on exit."""
|
||||
if (
|
||||
seconds is None
|
||||
or seconds <= 0
|
||||
or threading.current_thread() is not threading.main_thread()
|
||||
or not all(hasattr(signal, name) for name in ("SIGALRM", "ITIMER_REAL", "setitimer"))
|
||||
):
|
||||
yield
|
||||
return
|
||||
|
||||
def expired(signum, frame):
|
||||
raise ExtractionTimeout("wallclock_timeout")
|
||||
|
||||
previous = signal.getsignal(signal.SIGALRM)
|
||||
installed = False
|
||||
try:
|
||||
try:
|
||||
signal.signal(signal.SIGALRM, expired)
|
||||
installed = True
|
||||
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||
except ValueError:
|
||||
# Unsupported signal operations must not prevent extraction.
|
||||
if installed:
|
||||
signal.signal(signal.SIGALRM, previous)
|
||||
installed = False
|
||||
yield
|
||||
finally:
|
||||
if installed:
|
||||
try:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
finally:
|
||||
signal.signal(signal.SIGALRM, previous)
|
||||
|
||||
|
||||
class Git:
|
||||
def __init__(self, path: Path, *, runner=None, limit_bytes: int | None = None, deadline_seconds: float | None = None, _now=None):
|
||||
self._now = _now or time.monotonic
|
||||
self.started = self._now()
|
||||
self.deadline_seconds = deadline_seconds
|
||||
self.path = path
|
||||
self.runner = runner or subprocess.run
|
||||
self.bytes_read = 0
|
||||
self.files_read = 0
|
||||
self.last_blob_bytes = 0
|
||||
self.limit_bytes = limit_bytes
|
||||
|
||||
def command_timeout(self) -> float:
|
||||
if self.deadline_seconds is None:
|
||||
return 180
|
||||
remaining = self.deadline_seconds - (self._now() - self.started)
|
||||
if remaining <= 0:
|
||||
raise ExtractionTimeout("extraction_timeout")
|
||||
return min(180, remaining)
|
||||
|
||||
def run(self, *args: str, allowed=(0,)) -> str:
|
||||
timeout = self.command_timeout()
|
||||
try:
|
||||
done = self.runner(
|
||||
["git", "-c", "core.quotePath=false", "-C", str(self.path), *args],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise ExtractionTimeout("extraction_timeout") from None
|
||||
except ExtractionTimeout:
|
||||
raise
|
||||
except Exception:
|
||||
raise InstrumentFailure("git_launch_failed") from None
|
||||
if done.returncode not in allowed:
|
||||
raise InstrumentFailure("git_exit_" + str(done.returncode))
|
||||
if self.limit_bytes is not None and self.disk_bytes() > self.limit_bytes:
|
||||
raise InstrumentFailure("clone_size_cap")
|
||||
self.last_blob_bytes = len(done.stdout)
|
||||
return done.stdout.decode("utf-8", errors="replace")
|
||||
|
||||
def disk_bytes(self) -> int:
|
||||
# git churns .git/objects/pack (incl. transient *.rev) during on-demand blob
|
||||
# fetches, so a path rglob just listed can vanish before stat(): skip it.
|
||||
total = 0
|
||||
for p in self.path.rglob("*"):
|
||||
try:
|
||||
if p.is_file() and not p.is_symlink():
|
||||
total += p.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
def parents(self, sha: str) -> list[str]:
|
||||
return self.run("rev-list", "--parents", "-n", "1", sha).split()[1:]
|
||||
|
||||
def blob(self, sha: str, path: str) -> str | None:
|
||||
# Missing is established by a successful tree read, never by swallowing a show error.
|
||||
if not self.run("ls-tree", sha, "--", path).strip():
|
||||
return None
|
||||
text = self.run("show", f"{sha}:{path}")
|
||||
if Path(path).suffix in LANGUAGES:
|
||||
self.files_read += 1
|
||||
self.bytes_read += self.last_blob_bytes
|
||||
return text
|
||||
|
||||
def stats(self, base: str, head: str) -> list[tuple[str, int, int]]:
|
||||
rows = []
|
||||
for line in self.run("diff", "--no-renames", "--numstat", base, head, "--").splitlines():
|
||||
added, removed, path = line.split("\t", 2)
|
||||
rows.append((path, int(added) if added.isdigit() else 0, int(removed) if removed.isdigit() else 0))
|
||||
return rows
|
||||
|
||||
def hunks(self, base: str, head: str, path: str) -> list[tuple[int, int, int, int]]:
|
||||
diff = self.run("diff", "--no-ext-diff", "--no-renames", "-U0", base, head, "--", path)
|
||||
return [(int(m[1]), int(m[2] or 1), int(m[3]), int(m[4] or 1)) for m in HUNK.finditer(diff)]
|
||||
|
||||
def ancestor(self, older: str, newer: str) -> bool:
|
||||
# rev-list avoids interpreting a command failure as a negative ancestry result.
|
||||
return not self.run("rev-list", "--max-count=1", older, "--not", newer).strip()
|
||||
|
||||
def blame(self, revision: str, path: str, line: int) -> tuple[str, str, int]:
|
||||
text = self.run("blame", "-w", "-M", "-C", "--porcelain", f"-L{line},{line}", revision, "--", path)
|
||||
header = text.splitlines()[0].split()
|
||||
if len(header) < 3 or not SHA.fullmatch(header[0]):
|
||||
raise InstrumentFailure("invalid_blame")
|
||||
filename = next((row[9:] for row in text.splitlines() if row.startswith("filename ")), path)
|
||||
if filename.startswith('"'):
|
||||
filename = json.loads(filename)
|
||||
return header[0], filename, int(header[1])
|
||||
|
||||
|
||||
def functions(path: str, text: str) -> list[tuple[str, int, int]]:
|
||||
"""Conservative syntax boundaries. No scanner import; unresolved sites are rejected."""
|
||||
if Path(path).suffix == ".py":
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
raise Rejected("function_parse") from None
|
||||
return [
|
||||
(n.name, n.lineno, n.end_lineno or n.lineno) for n in ast.walk(tree) if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef)
|
||||
]
|
||||
# Blank comments and quoted strings before brace matching. Keep newlines/offsets.
|
||||
masked = re.sub(
|
||||
r'/\*.*?\*/|//[^\n]*|\#[^\n]*|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|`(?:\\.|[^`\\])*`',
|
||||
lambda m: "".join("\n" if c == "\n" else " " for c in m[0]),
|
||||
text,
|
||||
flags=re.S,
|
||||
)
|
||||
declaration = re.compile(
|
||||
r"\bfunction\s*&?\s*(\w+)\s*\([^)]*\)\s*(?::[^\n{]+)?\s*\{|"
|
||||
r"\b(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|\w+)\s*=>\s*\{|"
|
||||
r"\b([\w$]+)\s*\([^;{}]*\)\s*(?:throws\s+[\w., ]+)?\s*\{"
|
||||
)
|
||||
found = []
|
||||
for match in declaration.finditer(masked):
|
||||
name = next(g for g in match.groups() if g)
|
||||
if name in {"if", "while", "for", "switch", "catch", "with", "synchronized"}:
|
||||
continue
|
||||
depth = 1
|
||||
end = match.end()
|
||||
while end < len(masked) and depth:
|
||||
depth += (masked[end] == "{") - (masked[end] == "}")
|
||||
end += 1
|
||||
if depth == 0:
|
||||
found.append((name, masked.count("\n", 0, match.start()) + 1, masked.count("\n", 0, end) + 1))
|
||||
return found
|
||||
|
||||
|
||||
def enclosing(path: str, text: str, line: int, *, global_ok=False) -> tuple[str, int, int] | None:
|
||||
spans = [s for s in functions(path, text) if s[1] <= line <= s[2]]
|
||||
if spans:
|
||||
return min(spans, key=lambda s: s[2] - s[1])
|
||||
return ("<global>", 1, max(1, len(text.splitlines()))) if global_ok else None
|
||||
|
||||
|
||||
def normalized(text: str) -> str:
|
||||
return re.sub(r"\s+", "", text)
|
||||
|
||||
|
||||
def site_at(git, revision: str, path: str, name: str):
|
||||
text = git.blob(revision, path)
|
||||
if text is None:
|
||||
return None
|
||||
spans = [("<global>", 1, max(1, len(text.splitlines())))] if name == "<global>" else [s for s in functions(path, text) if s[0] == name]
|
||||
if len(spans) != 1:
|
||||
return None
|
||||
return text, spans[0]
|
||||
|
||||
|
||||
def fix_sites(git, fix: str, parent: str, *, global_ok=False) -> list[dict]:
|
||||
sites = []
|
||||
for path, _, _ in git.stats(parent, fix):
|
||||
if Path(path).suffix not in LANGUAGES:
|
||||
continue
|
||||
text = git.blob(parent, path)
|
||||
if text is None:
|
||||
continue
|
||||
lines = text.splitlines()
|
||||
for start, count, _, _ in git.hunks(parent, fix, path):
|
||||
for line in range(max(1, start), max(1, start) + max(1, count)):
|
||||
if line > len(lines):
|
||||
continue
|
||||
span = enclosing(path, text, line, global_ok=global_ok)
|
||||
if span is None:
|
||||
continue
|
||||
sink_line = line if count else span[1]
|
||||
sink = lines[sink_line - 1]
|
||||
if not sink.strip() or sink.lstrip().startswith(("#", "//", "/*", "*")):
|
||||
continue
|
||||
key = (path, span[0])
|
||||
if any((s["path"], s["function"]) == key for s in sites):
|
||||
continue
|
||||
sites.append({"path": path, "function": span[0], "line": sink_line, "text": sink, "absence": count == 0, "span": span})
|
||||
return sites
|
||||
|
||||
|
||||
def version_bracket(git, candidate: Candidate, revision: str) -> bool:
|
||||
if not candidate.first_affected:
|
||||
return True
|
||||
tags = git.run("tag", "--sort=version:refname").splitlines()
|
||||
matching = [t for t in tags if t.removeprefix("v") == candidate.first_affected]
|
||||
if len(matching) != 1:
|
||||
return False
|
||||
affected = matching[0]
|
||||
earlier = [t for t in tags[: tags.index(affected)] if git.ancestor(t, affected)]
|
||||
return bool(earlier) and git.ancestor(revision, affected) and not git.ancestor(revision, earlier[-1])
|
||||
|
||||
|
||||
def change_files(git, base: str, head: str) -> list[dict]:
|
||||
return [
|
||||
{"path": path, "head_lines": [[max(1, new), max(1, new + max(count, 1) - 1)] for _, _, new, count in git.hunks(base, head, path)]}
|
||||
for path, _, _ in git.stats(base, head)
|
||||
]
|
||||
|
||||
|
||||
def introducing(git, candidate: Candidate) -> dict:
|
||||
from collections import Counter
|
||||
|
||||
if git.parents(candidate.fix) != [candidate.parent]:
|
||||
raise Rejected("fix_parents")
|
||||
sites = fix_sites(git, candidate.fix, candidate.parent, global_ok=candidate.family == "config_secrets")
|
||||
reasons = Counter()
|
||||
|
||||
def record(method, base, head, path, name):
|
||||
resolved = site_at(git, head, path, name)
|
||||
if resolved is None:
|
||||
return None
|
||||
return {
|
||||
"kind": "vulnerability",
|
||||
"base": base,
|
||||
"head": head,
|
||||
"method": method,
|
||||
"family": candidate.family,
|
||||
"function": name,
|
||||
"path": path,
|
||||
"function_lines": list(resolved[1][1:]),
|
||||
"files": change_files(git, base, head),
|
||||
"post_cutoff": candidate.post_cutoff,
|
||||
"advisory": candidate.advisory,
|
||||
"bias": BIAS[method],
|
||||
"fallback_reasons": dict(reasons),
|
||||
}
|
||||
|
||||
for site in sites:
|
||||
revision, path, line = candidate.parent, site["path"], site["line"]
|
||||
for _ in range(6): # initial blame plus at most five walk-back steps
|
||||
blamed, blamed_path, blamed_line = git.blame(revision, path, line)
|
||||
parents = git.parents(blamed)
|
||||
if len(parents) != 1:
|
||||
reasons["merge_or_root"] += 1
|
||||
break
|
||||
base = parents[0]
|
||||
head_site = site_at(git, blamed, blamed_path, site["function"])
|
||||
if head_site is None:
|
||||
reasons["unresolved_function"] += 1
|
||||
break
|
||||
parent_text = git.blob(base, blamed_path)
|
||||
if parent_text is not None and site["function"] != "<global>":
|
||||
matching = [span for span in functions(blamed_path, parent_text) if span[0] == site["function"]]
|
||||
if len(matching) > 1:
|
||||
reasons["ambiguous_parent"] += 1
|
||||
break
|
||||
old_site = site_at(git, base, blamed_path, site["function"])
|
||||
occurrences = []
|
||||
if old_site:
|
||||
text, (_, start, end) = old_site
|
||||
occurrences = [
|
||||
i for i, value in enumerate(text.splitlines(), 1) if start <= i <= end and normalized(value) == normalized(site["text"])
|
||||
]
|
||||
if occurrences:
|
||||
reasons["text_present"] += 1
|
||||
revision, path, line = base, blamed_path, occurrences[0]
|
||||
continue
|
||||
stats = git.stats(base, blamed)
|
||||
if len(stats) > 50 or sum(a + d for _, a, d in stats) > 2000:
|
||||
reasons["size_cap"] += 1
|
||||
break
|
||||
if not version_bracket(git, candidate, blamed):
|
||||
reasons["version_bracket"] += 1
|
||||
break
|
||||
result = record("introducing", base, blamed, blamed_path, site["function"])
|
||||
if result:
|
||||
return result
|
||||
else:
|
||||
reasons["walkback_limit"] += 1
|
||||
for site in sites:
|
||||
# Numeric function extent implements log -L for all five languages, including
|
||||
# languages for which Git has no built-in :function: driver.
|
||||
_, start, end = site["span"]
|
||||
history = git.run("log", "--format=%H", "--no-patch", f"-L{start},{end}:{site['path']}", candidate.parent).splitlines()
|
||||
if not history:
|
||||
continue
|
||||
latest = history[0]
|
||||
parents = git.parents(latest)
|
||||
if len(parents) != 1:
|
||||
continue
|
||||
result = record("last_touch", parents[0], candidate.parent, site["path"], site["function"])
|
||||
if result:
|
||||
return result
|
||||
raise Rejected("no_range")
|
||||
|
||||
|
||||
SECURITY = re.compile(r"secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|exploit|sanitiz|escap", re.I)
|
||||
DEPENDENCY = re.compile(r"\bbump\b|\bupgrade .* from\b", re.I)
|
||||
MANIFESTS = {
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"yarn.lock",
|
||||
"pnpm-lock.yaml",
|
||||
"composer.json",
|
||||
"composer.lock",
|
||||
"pom.xml",
|
||||
"build.gradle",
|
||||
"build.gradle.kts",
|
||||
"gradle.lockfile",
|
||||
"pyproject.toml",
|
||||
"setup.py",
|
||||
"setup.cfg",
|
||||
"Pipfile",
|
||||
"Pipfile.lock",
|
||||
"poetry.lock",
|
||||
"uv.lock",
|
||||
}
|
||||
|
||||
|
||||
def generated_path(path: str) -> bool:
|
||||
return bool({"vendor", "node_modules", "dist"} & set(Path(path).parts)) or path.endswith(".min.js")
|
||||
|
||||
|
||||
def manifest(path: str) -> bool:
|
||||
name = Path(path).name
|
||||
return name in MANIFESTS or name.endswith(".lock") or bool(re.fullmatch(r"requirements[^/]*\.txt", name))
|
||||
|
||||
|
||||
def ordinary_rejection(row: dict, fixes: set[str], known_functions: set[tuple[str, str]]) -> str | None:
|
||||
if len(row["parents"]) != 1:
|
||||
return "merge_or_root"
|
||||
if row["head"] in fixes:
|
||||
return "advisory_fix"
|
||||
if SECURITY.search(row["message"]):
|
||||
return "security"
|
||||
if re.search(r"^Revert|This reverts commit", row["message"], re.I | re.M):
|
||||
return "revert"
|
||||
if re.search(r"dependabot|renovate|\[bot\]", row["author"], re.I) or DEPENDENCY.search(row["message"]):
|
||||
return "dependency"
|
||||
files = row["files"]
|
||||
if files and all(manifest(f["path"]) for f in files):
|
||||
return "manifest_only"
|
||||
source = [f for f in files if Path(f["path"]).suffix in LANGUAGES and not manifest(f["path"])]
|
||||
if not source:
|
||||
return "no_source"
|
||||
if all(f["generated"] or generated_path(f["path"]) for f in source):
|
||||
return "generated"
|
||||
if any((f["path"], name) in known_functions for f in files for name in f["functions"]):
|
||||
return "vulnerable_function"
|
||||
if any((f["path"], "<global>") in known_functions for f in files):
|
||||
return "vulnerable_function"
|
||||
if sum(f["added"] + f["removed"] for f in source) > 1000:
|
||||
return "size_cap"
|
||||
return None
|
||||
|
||||
|
||||
def generated(git, revision: str, path: str, text: str) -> bool:
|
||||
import fnmatch
|
||||
|
||||
if generated_path(path):
|
||||
return True
|
||||
header = "\n".join(text.splitlines()[:20])
|
||||
if re.search(r"(?im)^\s*(?:#|//|/\*|\*|<!--).*?(?:@generated|auto.?generated|generated\s+(?:by|file|code)|do not edit)", header):
|
||||
return True
|
||||
result = False
|
||||
parts = Path(path).parts
|
||||
for index in range(len(parts)):
|
||||
directory = "/".join(parts[:index])
|
||||
attributes = git.blob(revision, (directory + "/" if directory else "") + ".gitattributes") or ""
|
||||
relative = "/".join(parts[index:])
|
||||
for line in attributes.splitlines():
|
||||
tokens = line.split()
|
||||
if not tokens or tokens[0].startswith("#"):
|
||||
continue
|
||||
pattern = tokens[0].lstrip("/")
|
||||
target = relative if "/" in pattern else Path(relative).name
|
||||
if fnmatch.fnmatchcase(target, pattern):
|
||||
for token in tokens[1:]:
|
||||
if token in {"linguist-generated", "linguist-generated=true"}:
|
||||
result = True
|
||||
elif token in {"-linguist-generated", "!linguist-generated", "linguist-generated=false"}:
|
||||
result = False
|
||||
return result
|
||||
|
||||
|
||||
def ordinary_history(git, head: str) -> list[dict]:
|
||||
raw = git.run("log", "--first-parent", "--format=%H%x00%P%x00%ct%x00%an%x00%B%x1e", head)
|
||||
rows = []
|
||||
for entry in raw.split("\x1e"):
|
||||
if not entry.strip():
|
||||
continue
|
||||
fields = entry.strip().split("\x00")
|
||||
if len(fields) != 5 or not SHA.fullmatch(fields[0]):
|
||||
raise InstrumentFailure("invalid_history")
|
||||
sha, parents, timestamp, author, message = fields
|
||||
row = {"head": sha, "parents": parents.split(), "timestamp": int(timestamp), "author": author, "message": message, "files": []}
|
||||
rows.append(row)
|
||||
if len(row["parents"]) != 1:
|
||||
continue
|
||||
base = row["parents"][0]
|
||||
for path, added, removed in git.stats(base, sha):
|
||||
hunks = git.hunks(base, sha, path)
|
||||
info = {
|
||||
"path": path,
|
||||
"added": added,
|
||||
"removed": removed,
|
||||
"head_lines": [[max(1, n), max(1, n + max(c, 1) - 1)] for _, _, n, c in hunks],
|
||||
"functions": [],
|
||||
"generated": False,
|
||||
}
|
||||
if Path(path).suffix in LANGUAGES:
|
||||
# Inspect both sides: deleting a vulnerable function is still touching it.
|
||||
for revision, side in ((base, 0), (sha, 2)):
|
||||
text = git.blob(revision, path)
|
||||
if text is None:
|
||||
continue
|
||||
spans = functions(path, text)
|
||||
for hunk in hunks:
|
||||
start, count = hunk[side : side + 2]
|
||||
for name, lo, hi in spans:
|
||||
if max(1, start) <= hi and start + max(count, 1) - 1 >= lo:
|
||||
info["functions"].append(name)
|
||||
info["generated"] |= generated(git, revision, path, text)
|
||||
row["files"].append(info)
|
||||
return rows
|
||||
|
||||
|
||||
def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[str, str]], *, seed: int) -> tuple[list[dict], dict]:
|
||||
import random
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
counts = Counter()
|
||||
eligible = []
|
||||
for row in rows:
|
||||
reason = ordinary_rejection(row, fixes, known_functions)
|
||||
counts[reason or "eligible"] += 1
|
||||
if reason is None:
|
||||
eligible.append(row)
|
||||
if len(eligible) < 40:
|
||||
failure = Rejected("ordinary_floor")
|
||||
failure.counts = dict(counts)
|
||||
raise failure
|
||||
earliest, latest = min(r["timestamp"] for r in rows), max(r["timestamp"] for r in rows)
|
||||
cells = defaultdict(list)
|
||||
for row in eligible:
|
||||
third = min(2, 3 * (row["timestamp"] - earliest) // max(1, latest - earliest + 1))
|
||||
size = sum(f["added"] + f["removed"] for f in row["files"] if Path(f["path"]).suffix in LANGUAGES and not manifest(f["path"]))
|
||||
bucket = 0 if size <= 10 else 1 if size <= 50 else 2 if size <= 200 else 3
|
||||
cells[third, bucket].append(row)
|
||||
if {t for t, _ in cells} != {0, 1, 2}:
|
||||
failure = Rejected("time_thirds")
|
||||
failure.counts = dict(counts)
|
||||
raise failure
|
||||
# Largest-deficit rounding with the mandatory minimum of one per time third.
|
||||
# No cell can receive more draws than it has eligible commits.
|
||||
allocation = Counter()
|
||||
targets = {cell: 12 * len(items) / len(eligible) for cell, items in cells.items()}
|
||||
for third in range(3):
|
||||
cell = max(sorted(c for c in cells if c[0] == third), key=lambda c: targets[c])
|
||||
allocation[cell] = 1
|
||||
while sum(allocation.values()) < 12:
|
||||
cell = max(sorted(c for c in cells if allocation[c] < len(cells[c])), key=lambda c: targets[c] - allocation[c])
|
||||
allocation[cell] += 1
|
||||
rng = random.Random(seed)
|
||||
drawn = []
|
||||
for cell in sorted(cells):
|
||||
for row in rng.sample(sorted(cells[cell], key=lambda r: r["head"]), allocation[cell]):
|
||||
drawn.append(
|
||||
{
|
||||
"kind": "ordinary",
|
||||
"base": row["parents"][0],
|
||||
"head": row["head"],
|
||||
"method": "first_parent",
|
||||
"time_third": cell[0],
|
||||
"size_bucket": cell[1],
|
||||
"files": [{"path": f["path"], "head_lines": f["head_lines"]} for f in row["files"]],
|
||||
}
|
||||
)
|
||||
return drawn, dict(counts)
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"ghsa_id": "GHSA-test-test-test",
|
||||
"published_at": "2026-06-02T00:00:00Z",
|
||||
"type": "reviewed",
|
||||
"withdrawn_at": null,
|
||||
"cwes": [{"cwe_id": "CWE-89"}],
|
||||
"vulnerabilities": [{"package": {"ecosystem": "pip", "name": "fixture"}}],
|
||||
"references": ["https://github.com/fixture/project/commit/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]
|
||||
}
|
||||
@@ -1,860 +0,0 @@
|
||||
[
|
||||
{
|
||||
"id": "0",
|
||||
"repo": "0",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": [
|
||||
{
|
||||
"cwe": "CWE-89",
|
||||
"path": "app.py",
|
||||
"line": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1",
|
||||
"repo": "0",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"repo": "1",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "3",
|
||||
"repo": "1",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "4",
|
||||
"repo": "2",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5",
|
||||
"repo": "2",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "6",
|
||||
"repo": "3",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "7",
|
||||
"repo": "3",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "8",
|
||||
"repo": "4",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9",
|
||||
"repo": "4",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "10",
|
||||
"repo": "5",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "11",
|
||||
"repo": "5",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "12",
|
||||
"repo": "6",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "13",
|
||||
"repo": "6",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "14",
|
||||
"repo": "7",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "15",
|
||||
"repo": "7",
|
||||
"kind": "ordinary",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "16",
|
||||
"repo": "8",
|
||||
"kind": "vulnerability",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": [
|
||||
{
|
||||
"cwe": "CWE-89",
|
||||
"path": "app.py",
|
||||
"line": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "17",
|
||||
"repo": "8",
|
||||
"kind": "vulnerability",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": [
|
||||
{
|
||||
"cwe": "CWE-89",
|
||||
"path": "app.py",
|
||||
"line": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "18",
|
||||
"repo": "9",
|
||||
"kind": "vulnerability",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "19",
|
||||
"repo": "9",
|
||||
"kind": "vulnerability",
|
||||
"family": "injection",
|
||||
"method": "introducing",
|
||||
"post_cutoff": true,
|
||||
"label_check": "confirmed",
|
||||
"path": "app.py",
|
||||
"function": "handler",
|
||||
"files": [
|
||||
{
|
||||
"path": "app.py",
|
||||
"head_lines": [
|
||||
[
|
||||
10,
|
||||
12
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"instrument": {
|
||||
"head_verified": true,
|
||||
"base_verified": true,
|
||||
"changed_bytes": {
|
||||
"app.py": 20
|
||||
},
|
||||
"hook_bytes_read": 20,
|
||||
"hook_exit": 0
|
||||
},
|
||||
"push": {
|
||||
"quick_tier": [
|
||||
{
|
||||
"files": [
|
||||
"app.py"
|
||||
],
|
||||
"bytes_read": 20,
|
||||
"findings": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Freeze the staged, bisected p1 without running a guard, git, network or model.
|
||||
|
||||
Coordinator: python -m benchmarks.unseen.freeze --used-report USED-SET-REPORT.json
|
||||
The report is eligibility.UsedSet.report() output. Label-check counts are derived
|
||||
from retained vulnerability changes' label_check fields (missing means unchecked).
|
||||
A matching retry is a no-op; any changed frozen artifact requires a new pool id.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from .bisect_guard import read_pair
|
||||
from .draft import manifest
|
||||
from .eligibility import SOURCES
|
||||
from .journal import digest, sync_directory
|
||||
|
||||
|
||||
def count(value) -> int:
|
||||
if type(value) is not int or value < 0:
|
||||
raise ValueError("invalid_count")
|
||||
return value
|
||||
|
||||
|
||||
def sha256(value) -> str:
|
||||
if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value):
|
||||
raise ValueError("invalid_digest")
|
||||
return value
|
||||
|
||||
|
||||
def metadata(used_report: dict) -> dict:
|
||||
"""Only known source labels and numeric counts can enter the public record."""
|
||||
try:
|
||||
used_digest = sha256(used_report["sha256"])
|
||||
sources = used_report["sources"]
|
||||
if not sources or set(sources) - {*SOURCES, "memory_local", "memory_s3"}:
|
||||
raise ValueError
|
||||
return {"used_set_digest": used_digest, "source_counts": {k: count(v["count"]) for k, v in sources.items()}}
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
raise ValueError("invalid_used_report") from None
|
||||
|
||||
|
||||
def verify_receipt(receipt: dict, public: dict, private: dict) -> None:
|
||||
try:
|
||||
if type(receipt["guard_result"]) is not int or receipt["guard_result"] != 0:
|
||||
raise ValueError
|
||||
if sha256(receipt["draft_digest"]) != digest(public) or sha256(receipt["private_draft_digest"]) != digest(private):
|
||||
raise ValueError
|
||||
if count(receipt["input_count"]) != len(public.get("repository", [])) + count(receipt["dropped_count"]):
|
||||
raise ValueError
|
||||
except (ValueError, KeyError, TypeError):
|
||||
raise ValueError("invalid_guard_receipt") from None
|
||||
|
||||
|
||||
def frozen_text(data: dict) -> bytes:
|
||||
# Reuse the draft serializer, changing only the status. Hash its parsed value,
|
||||
# not the TOML bytes: table/key order and whitespace are not pool identity.
|
||||
text = manifest(data.get("repository", []), data["seed"])
|
||||
return text.replace('status = "draft"', 'status = "frozen"', 1).encode("utf-8")
|
||||
|
||||
|
||||
def matches(path: Path, payload: bytes) -> bool:
|
||||
try:
|
||||
if path.suffix == ".toml":
|
||||
return digest(tomllib.loads(path.read_text(encoding="utf-8"))) == digest(tomllib.loads(payload.decode("utf-8")))
|
||||
return digest(json.loads(path.read_bytes())) == digest(json.loads(payload))
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def run(root: Path, *, used_report: dict) -> dict:
|
||||
directory = root / "benchmarks/unseen"
|
||||
record = metadata(used_report)
|
||||
public, private = read_pair(directory / "draft-p1.toml", directory / "private/draft-p1.toml")
|
||||
try:
|
||||
receipt = json.loads((directory / "bisect-p1.json").read_bytes())
|
||||
except (OSError, ValueError):
|
||||
raise ValueError("invalid_guard_receipt") from None
|
||||
verify_receipt(receipt, public, private)
|
||||
if not public.get("repository"):
|
||||
raise ValueError("empty_pool")
|
||||
labels = dict.fromkeys(("confirmed", "refactor", "unclear", "unchecked"), 0)
|
||||
hidden = {r["id"]: r for r in private.get("repository", [])}
|
||||
try:
|
||||
for pointer in public["repository"]:
|
||||
row = hidden[pointer["id"]] if pointer.get("private") else pointer
|
||||
for change in row["changes"]:
|
||||
if change["kind"] == "vulnerability":
|
||||
labels[change.get("label_check", "unchecked")] += 1
|
||||
except (KeyError, TypeError):
|
||||
raise ValueError("invalid_label_check") from None
|
||||
try:
|
||||
protocol = (directory / "protocol-p1.md").read_bytes()
|
||||
if not protocol:
|
||||
raise ValueError
|
||||
except (OSError, ValueError):
|
||||
raise ValueError("invalid_protocol") from None
|
||||
public_bytes, private_bytes = frozen_text(public), frozen_text(private)
|
||||
record.update(
|
||||
guard_result=0,
|
||||
dropped_count=receipt["dropped_count"],
|
||||
label_check_counts=labels,
|
||||
freeze_digest=digest(tomllib.loads(public_bytes.decode("utf-8"))),
|
||||
protocol_digest=hashlib.sha256(protocol).hexdigest(),
|
||||
)
|
||||
outputs = (
|
||||
(directory / "private/pool-p1.toml", private_bytes),
|
||||
(directory / "pool-p1.toml", public_bytes),
|
||||
(directory / "freeze-p1.json", (json.dumps(record, sort_keys=True, indent=2) + "\n").encode()),
|
||||
)
|
||||
# Check EVERY existing artifact before writing anything. The record is the
|
||||
# completion marker and is written last; matching interrupted writes can resume.
|
||||
if any(path.exists() and not matches(path, payload) for path, payload in outputs):
|
||||
raise FileExistsError("frozen_pool_differs_use_new_pool")
|
||||
for path, payload in outputs:
|
||||
try:
|
||||
with path.open("xb") as handle:
|
||||
os.fchmod(handle.fileno(), 0o600)
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
sync_directory(path.parent)
|
||||
except FileExistsError:
|
||||
if not matches(path, payload):
|
||||
raise FileExistsError("frozen_pool_differs_use_new_pool") from None
|
||||
return record
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--used-report", type=Path, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
report = run(args.root, used_report=json.loads(args.used_report.read_bytes()))
|
||||
except Exception:
|
||||
print(json.dumps({"status": "freeze_refused"}))
|
||||
return 1
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,184 +0,0 @@
|
||||
"""Private, append-only sourcing checkpoints. No identity-bearing data leaves this directory."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
STAGES = ("advisories", "resolution", "eligibility", "extraction", "completion")
|
||||
|
||||
|
||||
def encoded(value) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
|
||||
|
||||
def digest(value) -> str:
|
||||
return hashlib.sha256(encoded(value)).hexdigest()
|
||||
|
||||
|
||||
class ResumeMismatch(ValueError):
|
||||
def __init__(self, parameters):
|
||||
self.parameters = parameters
|
||||
super().__init__("resume_inputs_changed")
|
||||
|
||||
|
||||
class Stopped(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def external(root: Path, path: Path) -> Path:
|
||||
path = path.expanduser().resolve()
|
||||
# Also reject the actual checkout when a different --root is supplied.
|
||||
if any(path.is_relative_to(p.resolve()) for p in (root, Path(__file__).resolve().parents[2])):
|
||||
raise ValueError("cache_inside_repository")
|
||||
return path
|
||||
|
||||
|
||||
def atomic(path: Path, payload: bytes) -> None:
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
with temporary.open("wb") as handle:
|
||||
os.chmod(temporary, 0o600)
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
temporary.replace(path)
|
||||
sync_directory(path.parent)
|
||||
|
||||
|
||||
def sync_directory(path: Path) -> None:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
class Journal:
|
||||
def __init__(self, path: Path, *, max_hours=None, clock=None, rate_limit=None):
|
||||
if max_hours is not None and (not math.isfinite(max_hours) or max_hours < 0):
|
||||
raise ValueError("invalid_max_hours")
|
||||
self.path = path
|
||||
self.clock = clock or time.monotonic
|
||||
self.started = self.clock()
|
||||
self.max_seconds = None if max_hours is None else max_hours * 3600
|
||||
self.rate_limit = rate_limit or (lambda: {})
|
||||
self.rows = {stage: {} for stage in STAGES}
|
||||
self.elapsed_before = 0.0
|
||||
self.last = None
|
||||
self.previous_rate = {}
|
||||
|
||||
@contextmanager
|
||||
def open(self, parameters):
|
||||
self.path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with (self.path / "LOCK").open("a") as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise ValueError("journal_busy") from None
|
||||
metadata = self.path / "inputs.json"
|
||||
if metadata.exists():
|
||||
old = json.loads(metadata.read_bytes())
|
||||
# Only current, fixed parameter labels can enter stdout.
|
||||
changed = [key for key in parameters if old.get(key) != parameters[key]]
|
||||
if old.keys() != parameters.keys():
|
||||
changed.append("parameter_schema")
|
||||
if changed:
|
||||
raise ResumeMismatch(changed)
|
||||
else:
|
||||
if any(self.path.glob("*.jsonl")):
|
||||
raise ValueError("journal_missing_inputs")
|
||||
atomic(metadata, encoded(parameters))
|
||||
for stage in STAGES:
|
||||
path = self.path / (stage + ".jsonl")
|
||||
if not path.exists():
|
||||
continue
|
||||
with path.open("rb") as handle:
|
||||
for line in handle:
|
||||
# Fail closed on torn/corrupt writes; never silently lose a completed unit.
|
||||
if not line.endswith(b"\n"):
|
||||
raise ValueError("journal_incomplete_record")
|
||||
row = json.loads(line)
|
||||
if row["id"] in self.rows[stage]:
|
||||
raise ValueError("journal_duplicate_unit")
|
||||
self.rows[stage][row["id"]] = row["value"]
|
||||
if row["elapsed_seconds"] >= self.elapsed_before:
|
||||
self.elapsed_before = row["elapsed_seconds"]
|
||||
self.last = {"stage": stage, "id": row["id"]}
|
||||
self.previous_rate = row["rate_limit"]
|
||||
self.progress("sourcing")
|
||||
yield self
|
||||
|
||||
def boundary(self):
|
||||
reason = None
|
||||
if (self.path / "STOP").exists():
|
||||
reason = "STOP"
|
||||
elif self.max_seconds is not None and self.clock() - self.started >= self.max_seconds:
|
||||
reason = "max_hours"
|
||||
if reason:
|
||||
self.progress("stopped", reason=reason)
|
||||
raise Stopped(reason)
|
||||
|
||||
def elapsed(self):
|
||||
return self.elapsed_before + self.clock() - self.started
|
||||
|
||||
def progress(self, status, **extra):
|
||||
atomic(
|
||||
self.path / "progress.json",
|
||||
encoded(
|
||||
{
|
||||
"status": status,
|
||||
"counts": {stage: len(rows) for stage, rows in self.rows.items()},
|
||||
"last_unit": self.last,
|
||||
"rate_limit": self.rate_limit() or self.previous_rate,
|
||||
"elapsed_seconds": self.elapsed(),
|
||||
**extra,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
def put(self, stage, key, value):
|
||||
if key in self.rows[stage]:
|
||||
raise ValueError("journal_duplicate_unit")
|
||||
row = {"id": key, "value": value, "elapsed_seconds": self.elapsed(), "rate_limit": self.rate_limit() or self.previous_rate}
|
||||
path = self.path / (stage + ".jsonl")
|
||||
with path.open("ab") as handle:
|
||||
os.chmod(path, 0o600)
|
||||
handle.write(encoded(row) + b"\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
sync_directory(self.path)
|
||||
self.rows[stage][key] = value
|
||||
self.last = {"stage": stage, "id": key}
|
||||
self.progress("sourcing")
|
||||
|
||||
|
||||
class ResolvedGitHub:
|
||||
"""Persist successful repository lookups and 404s used by both eligibility and the fix index."""
|
||||
|
||||
def __init__(self, github, journal):
|
||||
self.github, self.journal = github, journal
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.github, name)
|
||||
|
||||
def get_repo(self, name):
|
||||
from .eligibility import RepositoryNotFound, normalize
|
||||
|
||||
key = normalize(name)
|
||||
if key not in self.journal.rows["resolution"]:
|
||||
self.journal.boundary()
|
||||
try:
|
||||
value = self.github.get_repo(key)
|
||||
except RepositoryNotFound:
|
||||
value = None
|
||||
self.journal.put("resolution", key, value)
|
||||
value = self.journal.rows["resolution"][key]
|
||||
if value is None:
|
||||
raise RepositoryNotFound("not_found")
|
||||
return value
|
||||
@@ -1,88 +0,0 @@
|
||||
"""Append-only slice usage; a hash chain detects rewritten rows, not a rebuilt entire history.
|
||||
|
||||
Keep the published final digest to authenticate a history against truncation or wholesale replacement.
|
||||
Only opaque decision/experiment labels belong here; repository identities are never accepted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
FIELDS = {"slice", "decision", "purpose", "date", "result_digest", "freeze_digest"}
|
||||
PURPOSES = {"baseline", "exploratory", "informed", "qualifies"}
|
||||
ZERO = "0" * 64
|
||||
|
||||
|
||||
def digest(row: dict) -> str:
|
||||
return hashlib.sha256(json.dumps(row, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate(row: dict) -> None:
|
||||
if set(row) != FIELDS or row["purpose"] not in PURPOSES:
|
||||
raise ValueError("invalid_usage_fields")
|
||||
if type(row["slice"]) is not int or row["slice"] < 1:
|
||||
raise ValueError("invalid_slice")
|
||||
if not isinstance(row["decision"], str) or not re.fullmatch(r"[A-Za-z0-9_-]+", row["decision"]):
|
||||
raise ValueError("invalid_decision_label")
|
||||
for key in ("result_digest", "freeze_digest"):
|
||||
if not isinstance(row[key], str) or not re.fullmatch(r"[0-9a-f]{64}", row[key]):
|
||||
raise ValueError("invalid_digest")
|
||||
try:
|
||||
if date.fromisoformat(row["date"]).isoformat() != row["date"]:
|
||||
raise ValueError
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError("invalid_date") from None
|
||||
|
||||
|
||||
def _read(text: str) -> list[dict]:
|
||||
previous = ZERO
|
||||
rows = []
|
||||
if text and not text.endswith("\n"):
|
||||
raise ValueError("broken_digest_chain")
|
||||
for line in text.splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
stored = row.pop("digest")
|
||||
if row["previous"] != previous or digest(row) != stored:
|
||||
raise ValueError
|
||||
validate({k: v for k, v in row.items() if k != "previous"})
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
raise ValueError("broken_digest_chain") from None
|
||||
rows.append({**row, "digest": stored})
|
||||
previous = stored
|
||||
return rows
|
||||
|
||||
|
||||
def read(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open() as handle:
|
||||
fcntl.flock(handle, fcntl.LOCK_SH)
|
||||
return _read(handle.read())
|
||||
|
||||
|
||||
def append(path: Path, row: dict) -> str:
|
||||
validate(row)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a+") as handle:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
handle.seek(0)
|
||||
rows = _read(handle.read())
|
||||
if any(all(old[key] == row[key] for key in FIELDS) for old in rows):
|
||||
raise ValueError("duplicate_usage")
|
||||
if rows and rows[0]["freeze_digest"] != row["freeze_digest"]:
|
||||
raise ValueError("freeze_digest_changed")
|
||||
if row["purpose"] == "qualifies" and any(
|
||||
old["slice"] == row["slice"] and old["decision"] == row["decision"] and old["purpose"] == "informed" for old in rows
|
||||
):
|
||||
raise ValueError("slice_informed_decision")
|
||||
entry = {**row, "previous": rows[-1]["digest"] if rows else ZERO}
|
||||
sha = digest(entry)
|
||||
handle.write(json.dumps({**entry, "digest": sha}, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
return sha
|
||||
@@ -1,128 +0,0 @@
|
||||
"""Build anonymous coordinator proof records from completed dispatcher outputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def records(root):
|
||||
return [json.loads(path.read_text()) for path in sorted(root.glob("*.json")) if path.name not in {"progress.json", "inputs.json"}]
|
||||
|
||||
|
||||
def replay_proof(roots):
|
||||
lanes = {lane: records(root) for lane, root in roots.items()}
|
||||
if any(len(rows) != 5 for rows in lanes.values()):
|
||||
raise ValueError("proof requires exactly five completed changes per lane")
|
||||
keys = [{row["input_digest"] for row in rows} for rows in lanes.values()]
|
||||
if not all(key == keys[0] for key in keys):
|
||||
raise ValueError("lane input identities differ")
|
||||
quick = {}
|
||||
evidence = {}
|
||||
for lane, rows in lanes.items():
|
||||
quick[lane] = {}
|
||||
evidence[lane] = []
|
||||
for row in rows:
|
||||
inst = row.get("instrument", {})
|
||||
if (
|
||||
row.get("status") not in {"quick_only", "engine_covered"}
|
||||
or not inst.get("hook_bytes_read")
|
||||
or inst.get("suspect_fast")
|
||||
or inst.get("jvm_wall_seconds", 0) < 5
|
||||
):
|
||||
raise ValueError("proof instrument has unread input or implausible JVM time")
|
||||
quick[lane][row["input_digest"]] = digest([t.get("findings", []) for t in row["push"].get("quick_tier", [])])
|
||||
evidence[lane].append(
|
||||
{
|
||||
"input_digest": row["input_digest"],
|
||||
"seconds": inst["seconds"],
|
||||
"jvm_seconds": inst["jvm_wall_seconds"],
|
||||
"bytes": inst["hook_bytes_read"],
|
||||
"exit_code": inst["hook_exit"],
|
||||
}
|
||||
)
|
||||
equal = all(value == next(iter(quick.values())) for value in quick.values())
|
||||
if not equal:
|
||||
raise ValueError("quick-tier findings differ across lanes")
|
||||
return {"lanes": evidence, "quick_tier_equal": True, "quick_digests": quick, "cleanup": "completed-before-checkpoint"}
|
||||
|
||||
|
||||
def engine_proof(root, baseline):
|
||||
current = records(root)
|
||||
old = {digest([r["repo"], r["pin"]]): r for r in records(baseline) if "repo" in r and "pin" in r}
|
||||
current = [r for r in current if "repo" in r and "pin" in r and not all(u.get("status") == "unsupported" for u in r["units"])]
|
||||
if len(current) != 2:
|
||||
raise ValueError("engine proof requires exactly two pins")
|
||||
evidence = []
|
||||
|
||||
def stable(record):
|
||||
return [
|
||||
{key: u.get(key) for key in ("unit", "status", "questions_asked", "questions_completed", "witness_rows", "traces")}
|
||||
for u in record["units"]
|
||||
]
|
||||
|
||||
for row in current:
|
||||
key = digest([row["repo"], row["pin"]])
|
||||
previous = old[key]
|
||||
instruments = [u.get("instrument", {}) for u in row["units"]]
|
||||
if (
|
||||
not row.get("done")
|
||||
or not any(i.get("bytes", 0) > 0 for i in instruments)
|
||||
or not any(stage.get("seconds", 0) >= 5 for i in instruments for stage in i.get("jvm", []))
|
||||
):
|
||||
raise ValueError("engine proof lacks bytes or realistic JVM time")
|
||||
evidence.append(
|
||||
{
|
||||
"pin_digest": key,
|
||||
"seconds": row.get("seconds"),
|
||||
"record_digest": digest(stable(row)),
|
||||
"baseline_digest": digest(stable(previous)),
|
||||
"equal": stable(row) == stable(previous),
|
||||
"instrument": [
|
||||
{
|
||||
"bytes": u.get("instrument", {}).get("bytes", 0),
|
||||
"cpg_bytes": u.get("instrument", {}).get("cpg_bytes", 0),
|
||||
"jvm": [
|
||||
{key: stage.get(key) for key in ("seconds", "exit_code", "success", "heap_mb")}
|
||||
for stage in u.get("instrument", {}).get("jvm", [])
|
||||
],
|
||||
}
|
||||
for u in row["units"]
|
||||
],
|
||||
}
|
||||
)
|
||||
return {"pins": evidence, "differences": sum(not e["equal"] for e in evidence), "cleanup": "completed-before-checkpoint"}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("kind", choices=("engine", "replay"))
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--records", type=Path)
|
||||
parser.add_argument("--baseline", type=Path)
|
||||
parser.add_argument("--kind-records", type=Path)
|
||||
parser.add_argument("--ax-records", type=Path)
|
||||
parser.add_argument("--vm-records", type=Path)
|
||||
args = parser.parse_args()
|
||||
result = (
|
||||
engine_proof(args.records, args.baseline)
|
||||
if args.kind == "engine"
|
||||
else replay_proof({"kind": args.kind_records, "ax": args.ax_records, "vm": args.vm_records})
|
||||
)
|
||||
result.update(
|
||||
image=args.image, commit=subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip()
|
||||
)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,180 +0,0 @@
|
||||
# Pool p1: frozen scoring protocol
|
||||
|
||||
This protocol fixes the scoring, uncertainty estimates and adoption gate before any
|
||||
p1 replay. Its authority is the approved
|
||||
[unseen-repo-evaluation design](../../.kiro/specs/unseen-repo-evaluation/design.md),
|
||||
sections 3, 4 and 7 (including decisions 4.0, 4.0a and 7.3). The freeze record binds
|
||||
these exact UTF-8 file bytes by `protocol_digest`. Corrections require a new pool;
|
||||
neither p1 nor this protocol is revised in response to results.
|
||||
|
||||
## Population, slices and use
|
||||
|
||||
Design section 3 fixes three seeded, ecosystem- and `post_cutoff`-stratified slices
|
||||
of 100 repositories: one vulnerability change and 12 ordinary changes per
|
||||
repository, 100 + 1,200 = 1,300 changes per slice, 300 repositories and 3,900 changes
|
||||
overall. Guard-rejected candidates are dropped before freezing; actual retained
|
||||
counts are reported, with no replacement or silent claim that reduced slices meet
|
||||
the planned precision. Frozen slices are never resized.
|
||||
|
||||
Slice 1 supplies the baseline and G1 arms table. Slices 2 and 3 qualify the first
|
||||
two adoption decisions. Once spent, p2 is drawn by the same code and excludes p1.
|
||||
Every use is appended to `usage-p1.jsonl` with slice, decision or experiment id,
|
||||
purpose (`baseline`, `exploratory`, `informed`, `qualifies`), date, result digest and
|
||||
freeze digest. A slice that informed a change cannot qualify that change (3.1).
|
||||
|
||||
The label check reads every third `introducing` change before freezing, marking
|
||||
`confirmed`, `refactor` or `unclear`; report the confirmed share and catch on the
|
||||
confirmed subset. Report catch separately for `introducing` (the headline) and
|
||||
`last_touch` (a lower bound). The former inherits SZZ last-writer/refactor noise;
|
||||
the latter may already contain the flaw in base, making novelty-based silence
|
||||
correct even though scoring counts a miss (design 2.1).
|
||||
|
||||
## Replay arms
|
||||
|
||||
The default hook replay uses `--deadline 30 --engine inline --max-regions 500
|
||||
--mode advisory` (section 4). Retain the artifact and full text output, including
|
||||
every `Skipped:` line.
|
||||
|
||||
Decision 4.0 registers a second slice-1 arm with `--deadline 300` and every other
|
||||
flag unchanged. Commit its experiment manifest with
|
||||
`unseen: {pool: p1, slice: 1}` before its first replay. Report both arms to
|
||||
distinguish deadline misses from engine misses. This arm is a measurement;
|
||||
adopting a longer default deadline is a separate decision under section 7.2.
|
||||
|
||||
Decision 4.0a calibrates cluster overhead against native Docker runc. Use a
|
||||
deterministic paired sample of 20 changes by default, or all of a smaller proof
|
||||
manifest, at a 600-second calibration deadline. Record per-change stage times and
|
||||
median cluster/native slowdown with bootstrap 95% intervals separately for time
|
||||
through quick, engine build and engine queries. Incomplete timings or degraded
|
||||
scans invalidate calibration. Cluster arm deadlines are the 30/300-second
|
||||
deadlines multiplied by the median paired slowdown through quick, rounded to
|
||||
integer seconds; native deadlines stay at 30/300 seconds. Report applied
|
||||
deadlines and factors.
|
||||
|
||||
A deterministic native sample accompanies the cluster slice. Compare whether
|
||||
each push flags anything and report a Wilson 95% agreement interval. If its lower
|
||||
bound is below 95%, or any sample pair is missing or failed, flag the run and
|
||||
exclude cluster results from the baseline. Identify disagreements by opaque
|
||||
change IDs only. Calibration records contain counts, times and opaque digests.
|
||||
Docker checks its data-root free space against the unpacked image size upper
|
||||
bound plus 1 GiB before explicitly pulling; retain sanitized stderr tails and
|
||||
exit codes on failure (4.0a).
|
||||
|
||||
## Catch and false alarm
|
||||
|
||||
A vulnerability change is caught only if a finding matches its labelled family
|
||||
and head location (4.1). Engine findings carry the family; quick-rule CWEs map
|
||||
through the project's family CWE table. Match locations in the corresponding
|
||||
file using the following rules:
|
||||
|
||||
| Family | Required head location |
|
||||
| --- | --- |
|
||||
| `injection`, `path`, `deserialization`, `output_encoding`, `untrusted_destination` | Sink line in the vulnerable function, or within 10 lines of the head-side changed hunks |
|
||||
| `access_control` | The vulnerable function itself (the handler lacking the check), or within 10 lines of the head-side changed hunks |
|
||||
| `config_secrets` | The changed file and head-side changed lines ±10, at file scope (`<global>`) |
|
||||
|
||||
All three user-visible streams count: admitted defects, advisory engine findings
|
||||
and quick-rule matches. Record the stream responsible for each catch. A matching
|
||||
family at the wrong location does not count.
|
||||
|
||||
An ordinary change is a false alarm if **any** of those three streams contains at
|
||||
least one finding, of any family, anywhere (4.2). Report the admitted-defects-only
|
||||
alerts rate beside it. An ordinary change may contain an unknown vulnerability;
|
||||
adjudication does not change this definition. Adjudicate all flagged ordinary
|
||||
changes if there are 30 or fewer, otherwise every third, against the standard
|
||||
of protocol v2, and report the adjudicated share separately (design 2.2).
|
||||
|
||||
## Coverage and degradation
|
||||
|
||||
Assign exactly one state to every change (4.3):
|
||||
|
||||
| State | Definition |
|
||||
| --- | --- |
|
||||
| `instrument_failure` | No record, failed fetch, head commit not verified, or zero bytes of changed files read; also the `suspect_fast` failure path |
|
||||
| `unanalysable` | No changed file uses a language the hook covers (`language_not_covered`) |
|
||||
| `quick_only` | Quick read the changed files but no engine question completed on them |
|
||||
| `engine_covered` | At least one engine question completed on a changed file |
|
||||
|
||||
Prove input access: record checkout files and bytes, bytes of each changed file,
|
||||
JVM wall time and hook exit code. A replay faster than JVM startup is
|
||||
`suspect_fast`, never a clean zero. Failed fetches, missing commits and hook
|
||||
crashes yield failure records, never empty successes. Re-run an instrument
|
||||
failure once, then report it; it never counts as quiet or missed. Refuse to score
|
||||
a slice with more than 5% instrument failures after re-runs (design Error
|
||||
Handling). Preserve skipped/degraded coverage rather than interpreting silence
|
||||
as analysis. Report catch and false alarm overall and by coverage state, with
|
||||
failure counts explicit and failures excluded from outcome denominators.
|
||||
|
||||
## Intervals and planned precision
|
||||
|
||||
Use Wilson 95% intervals with z = 1.96 (section 3). For observed proportion p and
|
||||
denominator n, the centre is `(p + z²/(2n)) / (1 + z²/n)` and the half width is
|
||||
`z * sqrt(p*(1-p)/n + z²/(4n²)) / (1 + z²/n)`. The precision bar uses the larger
|
||||
distance from p to either bound, not just the interval's half width.
|
||||
|
||||
The design's exact minimum sample sizes are:
|
||||
|
||||
| Rate | Assumed p | Maximum distance | Required n |
|
||||
| --- | --- | --- | --- |
|
||||
| False alarm | 0.05 | 3 percentage points | 315 |
|
||||
| False alarm | 0.10 | 3 percentage points | 483 |
|
||||
| False alarm | 0.20 | 3 percentage points | 756 |
|
||||
| Catch | 0.10 | 10 percentage points | 62 |
|
||||
| Catch | 0.25 | 10 percentage points | 88 |
|
||||
| Catch | 0.40 | 10 percentage points | 97 |
|
||||
|
||||
At n = 100 and catch p = 0.4 the Wilson interval is [0.309, 0.498], within
|
||||
10 points. One vulnerability change per repository keeps those observations
|
||||
independent. The 12 ordinary changes within a repository are correlated. With
|
||||
m = 12 and assumed ICC = 0.05 the design effect `1 + (m - 1)*ICC` is 1.55;
|
||||
1,200 ordinary changes give effective n ≈774 ≥756. At p = 0.2 their plain Wilson
|
||||
interval is about ±2.3 points.
|
||||
|
||||
Report a repository-cluster bootstrap 95% interval beside the ordinary-change
|
||||
Wilson interval: resample whole repositories, retaining their within-repository
|
||||
changes. Measure and record ICC on slice 1. If ICC exceeds 0.05, increase the next
|
||||
pool's size, never the frozen slice's. Per-family intervals are reported but not
|
||||
sized; splitting 100 changes over seven families gives roughly ±20–25 points.
|
||||
The design leaves per-family BLOCK sizing to a later decision.
|
||||
|
||||
Report counts, denominators, rates and intervals overall and per family, method,
|
||||
post-cutoff group, stream, coverage state and confirmed subset (sections 3 and 7).
|
||||
Use actual observed counts, not planned sample sizes, for intervals.
|
||||
|
||||
## Fixed budgets and adoption gate
|
||||
|
||||
For score-bearing arms, report catch at false-alarm budgets of 1%, 5% and 10%
|
||||
(7.1). Choose the lowest score threshold whose ordinary-change false-alarm rate
|
||||
stays within the budget; report catch with its Wilson interval and realised
|
||||
false-alarm rate. A threshold selected on the same slice is `exploratory`. For
|
||||
qualification, fix it beforehand on an earlier slice or held-out folds and
|
||||
apply it unchanged. A binary arm has one operating point; report which budgets
|
||||
it meets. Do not substitute AUC for these operating points.
|
||||
|
||||
Registered experiments identify `unseen: {pool, slice}`. Write the unseen
|
||||
estimates beside held-out fold estimates, including each arm's catch at each
|
||||
budget and paired difference B − A with an exact McNemar test on the paired
|
||||
vulnerability changes (planned n = 100; 7.2). Record use in the ledger.
|
||||
|
||||
Adopt candidate B over production A only when both the existing fold rule and
|
||||
the untouched-slice gate pass (7.3):
|
||||
|
||||
1. At the 5% budget with B's threshold fixed beforehand, B's catch is not
|
||||
significantly lower than A's: exact two-sided McNemar on paired vulnerability
|
||||
changes, alpha 0.05.
|
||||
2. For a binary arm, B's false-alarm rate is not significantly higher than A's:
|
||||
exact two-sided McNemar on paired ordinary changes, alpha 0.05. Report the
|
||||
repository-cluster bootstrap interval for the false-alarm difference beside
|
||||
the test because those changes are clustered.
|
||||
|
||||
Write `unseen.gate: pass | regression | not_run`. No slice, a slice already
|
||||
`informed` for this change, or more than 5% instrument failures yields `not_run`;
|
||||
it blocks adoption just as `regression` does. Decision 4.0a also excludes flagged
|
||||
cluster/native agreement runs from baseline use. Failure to detect a significant
|
||||
regression is the specified gate, not evidence that the arms are equivalent.
|
||||
|
||||
Raw per-change artifacts remain outside the repository. Committed measurement
|
||||
records contain counts, rates, intervals and digests, never repository names or
|
||||
URLs. Public p1 names stay only in the permissive pool manifest; private entries
|
||||
remain in the gitignored private manifest and are bound through opaque pointers
|
||||
(design 1.4 and 6).
|
||||
@@ -1,247 +0,0 @@
|
||||
"""Dispatch frozen unseen changes or a five-change project-history proof."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.ax.batch import AXLane, DockerLane, Item, Workload, dispatch, write_json
|
||||
from benchmarks.unseen.score import coverage
|
||||
|
||||
|
||||
def make_item(change, arm, image, *, deadline_scale=1.0, lane="ax", deadline=None):
|
||||
if arm not in {"default", "long_deadline"}:
|
||||
raise ValueError("unknown replay arm")
|
||||
identity = hashlib.sha256(str(change["id"]).encode()).hexdigest()[:14]
|
||||
if not math.isfinite(deadline_scale) or deadline_scale <= 0:
|
||||
raise ValueError("deadline scale must be finite and positive")
|
||||
base = deadline if deadline is not None else 30 if arm == "default" else 300
|
||||
applied = round(base * (1 if lane == "docker" else deadline_scale))
|
||||
if applied < 1:
|
||||
raise ValueError("scaled deadline must be at least one second")
|
||||
spec = {
|
||||
**change,
|
||||
"id": identity,
|
||||
"hook_flags": ["--deadline", str(applied)],
|
||||
"image": image,
|
||||
"base_deadline_seconds": base,
|
||||
"deadline_seconds": applied,
|
||||
"deadline_scale": deadline_scale,
|
||||
"applied_deadline_scale": 1.0 if lane == "docker" else deadline_scale,
|
||||
}
|
||||
data = json.dumps(spec, sort_keys=True).encode()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
return Item(
|
||||
"replay-" + identity,
|
||||
{"SPEC_URL": data},
|
||||
"result.dat",
|
||||
("python3", "/app/benchmarks/unseen/replay_entry.py"),
|
||||
{},
|
||||
digest,
|
||||
{"SPEC_URL": "spec.dat"},
|
||||
)
|
||||
|
||||
|
||||
def validate_result(data, output):
|
||||
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
||||
members = archive.getmembers()
|
||||
required = {"push.json", "push.txt", "engine.json", "instrument.json"}
|
||||
if {m.name for m in members} != required or any(not m.isfile() for m in members):
|
||||
raise ValueError("replay result requires push, engine and instrument.json")
|
||||
payload = {m.name: archive.extractfile(m).read() for m in members}
|
||||
instrument = json.loads(payload["instrument.json"])
|
||||
push = json.loads(payload["push.json"])
|
||||
record = {"instrument": instrument, "push": push}
|
||||
record["status"] = coverage(record)
|
||||
# Transport/scheduler needs an explicit OOM signal, not a quiet failed replay.
|
||||
if "OutOfMemoryError" in json.dumps(push) or "JVM out of memory" in json.dumps(push):
|
||||
record["status"] = "oom"
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
for name, body in payload.items():
|
||||
(output / name).write_bytes(body)
|
||||
(output / "result.dat").write_bytes(data)
|
||||
write_json(output / "result.json", record)
|
||||
return record
|
||||
|
||||
|
||||
def proof_changes(root):
|
||||
"""Only the operator's project history; no pool data or fetches."""
|
||||
|
||||
def git(*args):
|
||||
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
changes = []
|
||||
for head in git("rev-list", "--no-merges", "--max-count=100", "HEAD", "--", "src").splitlines():
|
||||
base = git("rev-parse", head + "^")
|
||||
changed = git("diff", "--name-only", base, head).splitlines()
|
||||
if not any(name.startswith("src/") and name.endswith(".py") for name in changed):
|
||||
continue
|
||||
changes.append(
|
||||
{
|
||||
"id": hashlib.sha256(head.encode()).hexdigest(),
|
||||
"repository_url": "https://github.com/norandom/OpenUltraSAST.git",
|
||||
"repo": "project-proof",
|
||||
"kind": "ordinary",
|
||||
"base": base,
|
||||
"head": head,
|
||||
}
|
||||
)
|
||||
if len(changes) == 5:
|
||||
break
|
||||
if len(changes) != 5:
|
||||
raise ValueError("history lacks five source changes")
|
||||
return changes
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
|
||||
|
||||
def load_slice(path, slice_id, private_path=None):
|
||||
if path.suffix != ".toml":
|
||||
frozen = json.loads(path.read_text())
|
||||
changes = frozen["changes"]
|
||||
if frozen["sha256"] != hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest():
|
||||
raise ValueError("frozen changes digest mismatch")
|
||||
return changes
|
||||
if not path.name.startswith("pool-"):
|
||||
raise ValueError("replay requires a frozen pool, never a draft")
|
||||
pool = tomllib.loads(path.read_text())
|
||||
freeze = json.loads(path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json").read_text())
|
||||
if hashlib.sha256(canonical(pool)).hexdigest() != freeze["freeze_digest"]:
|
||||
raise ValueError("frozen pool digest mismatch")
|
||||
private_path = private_path or path.parent / "private" / path.name
|
||||
private = {}
|
||||
if private_path.exists():
|
||||
private = {row["id"]: row for row in tomllib.loads(private_path.read_text()).get("repository", [])}
|
||||
changes = []
|
||||
for public in pool.get("repository", []):
|
||||
if str(public["slice"]) != str(slice_id):
|
||||
continue
|
||||
row = public
|
||||
if public.get("private"):
|
||||
row = private.get(public["id"])
|
||||
if row is None:
|
||||
raise ValueError("missing private pool entry")
|
||||
payload = {"url": row["url"], "commits": [[c["base"], c["head"]] for c in row["changes"]]}
|
||||
if hashlib.sha256(canonical(payload)).hexdigest() != public["sha256"]:
|
||||
raise ValueError("private pool digest mismatch")
|
||||
for change in row["changes"]:
|
||||
identity = hashlib.sha256(canonical([public["id"], change["kind"], change["base"], change["head"]])).hexdigest()
|
||||
changes.append(
|
||||
{
|
||||
**change,
|
||||
"id": identity,
|
||||
"repo": public["id"],
|
||||
"repository_url": row["url"],
|
||||
"freeze_digest": freeze["freeze_digest"],
|
||||
"slice": slice_id,
|
||||
}
|
||||
)
|
||||
if not changes:
|
||||
raise ValueError("frozen slice has no changes")
|
||||
return changes
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True, help="frozen JSON: changes array plus sha256 of canonical changes JSON")
|
||||
parser.add_argument("--proof-history", action="store_true", help="write a five-change own-history manifest, then exit")
|
||||
parser.add_argument("--image")
|
||||
parser.add_argument("--private-manifest", type=Path)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--arm", choices=("default", "long_deadline"), default="default")
|
||||
parser.add_argument("--pool", default="p1")
|
||||
parser.add_argument("--slice", default="1")
|
||||
parser.add_argument("--out", type=Path)
|
||||
parser.add_argument("--lane", choices=("ax", "kind", "docker", "mixed"), default="mixed")
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument("--deadline-scale", type=float, default=1.0)
|
||||
group.add_argument("--calibration", type=Path)
|
||||
parser.add_argument("--native-sample", type=int, default=0)
|
||||
parser.add_argument("--docker-image-bytes", type=int, help="unpacked image size upper bound, required for VM pull")
|
||||
parser.add_argument("--ax-bin", default="ax")
|
||||
parser.add_argument("--kubeconfig", type=Path)
|
||||
parser.add_argument("--atespace", default="default")
|
||||
args = parser.parse_args()
|
||||
if args.proof_history:
|
||||
changes = proof_changes(Path.cwd())
|
||||
write_json(args.manifest, {"changes": changes, "sha256": hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest()})
|
||||
return 0
|
||||
if not args.image:
|
||||
parser.error("--image digest is required")
|
||||
changes = load_slice(args.manifest, args.slice, args.private_manifest)
|
||||
factor = args.deadline_scale
|
||||
if args.calibration:
|
||||
calibration = json.loads(args.calibration.read_text())
|
||||
if calibration.get("status") != "ok" or calibration.get("image") != args.image:
|
||||
parser.error("calibration must be successful and match the replay image")
|
||||
factor = calibration["quick_total"]["factor"]
|
||||
if args.native_sample < 0 or (args.native_sample and args.lane not in {"ax", "kind"}):
|
||||
parser.error("--native-sample requires an ax or kind cluster run and a positive K")
|
||||
items = [make_item(change, args.arm, args.image, deadline_scale=factor, lane=args.lane) for change in changes]
|
||||
if len({i.id for i in items}) != len(items):
|
||||
parser.error("duplicate change identity")
|
||||
args.out = args.out or Path.home() / "ousast-results/unseen" / args.pool / args.slice / args.arm
|
||||
if args.dry_run:
|
||||
print(json.dumps({"items": len(items), "input_digest": hashlib.sha256(canonical([i.input_digest for i in items])).hexdigest()}))
|
||||
return 0
|
||||
if args.out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
|
||||
parser.error("raw replay output must be outside the repository")
|
||||
failed = run_replays(changes, args, args.out, args.lane, deadline_scale=factor)
|
||||
if args.native_sample:
|
||||
failed |= run_replays(sample_changes(changes, args.native_sample), args, args.out / "native", "docker", deadline_scale=factor)
|
||||
return 2 if failed else 0
|
||||
|
||||
|
||||
def sample_changes(changes, count):
|
||||
if count < 1:
|
||||
raise ValueError("sample size must be positive")
|
||||
if len({str(c["id"]) for c in changes}) != len(changes):
|
||||
raise ValueError("duplicate change identity")
|
||||
return sorted(changes, key=lambda c: hashlib.sha256(str(c["id"]).encode()).digest())[:count]
|
||||
|
||||
|
||||
def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None):
|
||||
if out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
|
||||
raise ValueError("raw replay output must be outside the repository")
|
||||
items = [make_item(c, args.arm, args.image, deadline_scale=deadline_scale, lane=lane, deadline=deadline) for c in changes]
|
||||
write_json(
|
||||
out / "inputs.json",
|
||||
[{"item_id": item.id, "change": change, "input_digest": item.input_digest} for item, change in zip(items, changes, strict=True)],
|
||||
)
|
||||
maximum = max(
|
||||
max(json.loads(i.inputs["SPEC_URL"])["deadline_seconds"], json.loads(i.inputs["SPEC_URL"])["base_deadline_seconds"]) for i in items
|
||||
)
|
||||
workload = Workload(args.image, frozenset({"SPEC_URL", "RESULT_URL"}), maximum + 600, validate_result, kind="replay")
|
||||
lanes = ("ax", "ax", "docker") if lane == "mixed" else (lane,) * (2 if lane in {"ax", "kind"} else 1)
|
||||
original = {item.id: change for item, change in zip(items, changes, strict=True)}
|
||||
executors = {}
|
||||
for name in set(lanes):
|
||||
transport = (DockerLane if name == "docker" else AXLane)(args, workload)
|
||||
|
||||
def execute(item, output, attempt, transport=transport, name=name):
|
||||
spec = original[item.id]
|
||||
# Re-render on placement, including VM fallback: native is always unscaled.
|
||||
placed = make_item(spec, args.arm, args.image, deadline_scale=deadline_scale, lane=name, deadline=deadline)
|
||||
placed.id = item.id
|
||||
record = {**json.loads(placed.inputs["SPEC_URL"]), "lane": name, **transport(placed, output, attempt)}
|
||||
if (output / "result.dat").exists():
|
||||
key = f"unseen/{placed.input_digest}/{name}/result.dat"
|
||||
transport.store._put(key, (output / "result.dat").read_bytes())
|
||||
record["result_object"] = key
|
||||
return record
|
||||
|
||||
executors[name] = execute
|
||||
return dispatch(items, workload, out, executors, lanes=lanes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,224 +0,0 @@
|
||||
"""Credential-free replay actor. Standard library only; PUT is completion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "learn"))
|
||||
from engine_task_entry import transfer
|
||||
|
||||
CLUSTER_FLAGS = (
|
||||
"-Xms1700m -Xmx1700m -XX:MaxMetaspaceSize=256m -XX:ReservedCodeCacheSize=128m "
|
||||
"-XX:MaxDirectMemorySize=256m -Xss512k -XX:ActiveProcessorCount=2 -XX:+UseG1GC"
|
||||
)
|
||||
|
||||
|
||||
class ReplayFailure(Exception):
|
||||
"""Locally controlled diagnostic only."""
|
||||
|
||||
|
||||
def validate_spec(spec):
|
||||
parts = urlsplit(spec["repository_url"])
|
||||
if (
|
||||
parts.scheme != "https"
|
||||
or not parts.hostname
|
||||
or "." not in parts.hostname
|
||||
or parts.username
|
||||
or parts.password
|
||||
or parts.query
|
||||
or parts.fragment
|
||||
):
|
||||
raise ReplayFailure("repository must be public HTTPS")
|
||||
if parts.port not in (None, 443) or parts.hostname.endswith((".local", ".internal", ".localhost")):
|
||||
raise ReplayFailure("repository must be public HTTPS")
|
||||
import ipaddress
|
||||
|
||||
try:
|
||||
ipaddress.ip_address(parts.hostname)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise ReplayFailure("repository must use public DNS")
|
||||
if any(not re.fullmatch("[0-9a-f]{40,64}", spec[key]) for key in ("base", "head")):
|
||||
raise ReplayFailure("invalid revision")
|
||||
flags = spec.get("hook_flags", [])
|
||||
if flags and (len(flags) != 2 or flags[0] != "--deadline" or not re.fullmatch(r"[1-9][0-9]*", str(flags[1]))):
|
||||
raise ReplayFailure("unsupported hook flags")
|
||||
|
||||
|
||||
def main(*, work_root=Path("/tmp"), runner=subprocess.run, heap_profile="cluster", clock=time.monotonic):
|
||||
output = work_root / "out"
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
instrument = {
|
||||
"status": "instrument_failure",
|
||||
"head_verified": False,
|
||||
"base_verified": False,
|
||||
"changed_bytes": {},
|
||||
"checkout_files": 0,
|
||||
"checkout_bytes": 0,
|
||||
"hook_bytes_read": 0,
|
||||
"jvm_wall_seconds": 0,
|
||||
"jvm_time_basis": "scan build_seconds + query_seconds",
|
||||
"hook_exit": None,
|
||||
"timings": {},
|
||||
"heap_profile": heap_profile,
|
||||
}
|
||||
spec = {}
|
||||
step = "GET spec"
|
||||
started = clock()
|
||||
|
||||
def run(command, *, timeout=180, env=None):
|
||||
before = clock()
|
||||
result = runner(command, capture_output=True, text=True, timeout=timeout, env=env)
|
||||
instrument["timings"][step] = instrument["timings"].get(step, 0) + clock() - before
|
||||
if result.returncode:
|
||||
raise ReplayFailure(f"exit {result.returncode}")
|
||||
return result
|
||||
|
||||
try:
|
||||
spec_path = work_root / "spec.dat"
|
||||
transfer(os.environ["SPEC_URL"], spec_path, readiness_budget=60)
|
||||
step = "validate spec"
|
||||
spec = json.loads(spec_path.read_text())
|
||||
instrument["image"] = spec.get("image")
|
||||
validate_spec(spec)
|
||||
instrument["deadline_seconds"] = int(spec["hook_flags"][1]) if spec.get("hook_flags") else 30
|
||||
instrument["deadline_scale"] = spec.get("deadline_scale", 1.0)
|
||||
case = work_root / "case"
|
||||
case.mkdir()
|
||||
git = ["git", "-c", "core.hooksPath=/dev/null", "-c", "protocol.file.allow=never", "-C", str(case)]
|
||||
step = "git init"
|
||||
run([*git, "init", "-q"])
|
||||
run([*git, "remote", "add", "origin", spec["repository_url"]])
|
||||
# One total fetch budget, independent of the hook deadline.
|
||||
fetch_end = clock() + 180
|
||||
for key in ("head", "base"):
|
||||
step = "fetch " + key
|
||||
remaining = fetch_end - clock()
|
||||
if remaining <= 0:
|
||||
raise ReplayFailure("fetch deadline")
|
||||
run([*git, "fetch", "--no-tags", "--depth", "1", "origin", spec[key]], timeout=remaining)
|
||||
step = "verify " + key
|
||||
run([*git, "cat-file", "-e", spec[key] + "^{commit}"])
|
||||
instrument[key + "_verified"] = True
|
||||
step = "checkout"
|
||||
run([*git, "checkout", "--detach", spec["head"]])
|
||||
paths = run([*git, "ls-files", "-z"]).stdout.split("\0")
|
||||
for name in filter(None, paths):
|
||||
path = case / name
|
||||
if path.is_file() and not path.is_symlink():
|
||||
data = path.read_bytes()
|
||||
instrument["checkout_files"] += 1
|
||||
instrument["checkout_bytes"] += len(data)
|
||||
if instrument["checkout_bytes"] > 500 * 1024 * 1024:
|
||||
raise ReplayFailure("checkout size limit")
|
||||
step = "changed files"
|
||||
changed = run([*git, "diff", "--name-only", "--diff-filter=ACMRT", "-z", spec["base"], spec["head"]]).stdout.split("\0")
|
||||
for name in filter(None, changed):
|
||||
path = case / name
|
||||
if path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(case.resolve()):
|
||||
instrument["changed_bytes"][name] = len(path.read_bytes())
|
||||
if not instrument["checkout_bytes"] or not sum(instrument["changed_bytes"].values()):
|
||||
instrument["suspect_fast"] = True
|
||||
raise ReplayFailure("no changed bytes read")
|
||||
step = "pre-push"
|
||||
env = {
|
||||
key: value for key, value in os.environ.items() if key not in {"SPEC_URL", "RESULT_URL", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS"}
|
||||
}
|
||||
env["OPENULTRASAST_CPG_HEAP_MB"] = "1700" if heap_profile == "cluster" else "2560"
|
||||
env["JAVA_TOOL_OPTIONS"] = CLUSTER_FLAGS if heap_profile == "cluster" else "-Xmx2560m"
|
||||
command = [
|
||||
"ousast",
|
||||
"pre-push",
|
||||
str(case),
|
||||
"--base",
|
||||
spec["base"],
|
||||
"--head",
|
||||
spec["head"],
|
||||
"--artifact",
|
||||
str(output / "push.json"),
|
||||
*spec.get("hook_flags", []),
|
||||
]
|
||||
before = clock()
|
||||
completed = runner(
|
||||
command, capture_output=True, text=True, timeout=int(spec["hook_flags"][1]) + 30 if spec.get("hook_flags") else 60, env=env
|
||||
)
|
||||
instrument["timings"][step] = clock() - before
|
||||
instrument["hook_exit"] = completed.returncode
|
||||
(output / "push.txt").write_text(completed.stdout)
|
||||
if completed.returncode not in (0, 1):
|
||||
raise ReplayFailure(f"hook exit {completed.returncode}")
|
||||
step = "read push artifact"
|
||||
push = json.loads((output / "push.json").read_text())
|
||||
instrument["hook_bytes_read"] = sum(t.get("bytes_read", 0) for t in push.get("quick_tier", []))
|
||||
scans = push.get("scans", [])
|
||||
instrument["jvm_wall_seconds"] = sum(s.get("scan", {}).get(k, 0) for s in scans for k in ("build_seconds", "query_seconds"))
|
||||
uncovered = "language_not_covered" in json.dumps(push.get("admission", {}).get("coverage_reasons", []))
|
||||
partitions = [part for scan in scans for part in scan.get("scan", {}).get("partitions", [])]
|
||||
unsupported_only = (
|
||||
uncovered
|
||||
and instrument["hook_bytes_read"] == 0
|
||||
and bool(partitions)
|
||||
and all(part.get("status") == "unsupported" for part in partitions)
|
||||
)
|
||||
instrument["suspect_fast"] = bool(scans) and not unsupported_only and instrument["jvm_wall_seconds"] < 5
|
||||
if instrument["suspect_fast"] or (instrument["hook_bytes_read"] <= 0 and not uncovered):
|
||||
instrument["suspect_fast"] = True
|
||||
raise ReplayFailure("suspect_fast or unread hook input")
|
||||
engine = {
|
||||
"repository": spec["repository_url"],
|
||||
"revisions": {key: spec[key] for key in ("base", "head")},
|
||||
"scans": scans,
|
||||
"admission": push.get("admission", {}),
|
||||
}
|
||||
(output / "engine.json").write_text(json.dumps(engine))
|
||||
instrument["status"] = "ok"
|
||||
except Exception as exc:
|
||||
detail = (
|
||||
f"HTTP {exc.code}"
|
||||
if isinstance(exc, urllib.error.HTTPError)
|
||||
else str(exc)
|
||||
if isinstance(exc, ReplayFailure)
|
||||
else type(exc).__name__
|
||||
)
|
||||
instrument["failure"] = {"step": step, "status": detail}
|
||||
# A crash artifact must never be mistaken for a completed scan.
|
||||
if not (output / "push.json").exists():
|
||||
(output / "push.json").write_text("{}")
|
||||
if not (output / "push.txt").exists():
|
||||
(output / "push.txt").write_text("")
|
||||
if not (output / "engine.json").exists():
|
||||
(output / "engine.json").write_text("{}")
|
||||
instrument["seconds"] = clock() - started
|
||||
(output / "instrument.json").write_text(json.dumps(instrument))
|
||||
try:
|
||||
archive = work_root / "result.dat"
|
||||
with tarfile.open(archive, "w") as tar:
|
||||
for name in ("push.json", "push.txt", "engine.json", "instrument.json"):
|
||||
tar.add(output / name, arcname=name)
|
||||
request = urllib.request.Request(
|
||||
os.environ["RESULT_URL"], data=archive.read_bytes(), method="PUT", headers={"Content-Type": "application/x-tar"}
|
||||
)
|
||||
transfer(request, readiness_budget=60)
|
||||
except Exception as exc:
|
||||
detail = f"HTTP {exc.code}" if isinstance(exc, urllib.error.HTTPError) else type(exc).__name__
|
||||
print("PUT result: " + detail, file=sys.stderr)
|
||||
return 1
|
||||
return int(instrument["status"] != "ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--heap-profile", choices=("cluster", "vm"), default="cluster")
|
||||
raise SystemExit(main(heap_profile=parser.parse_args().heap_profile))
|
||||
@@ -1,363 +0,0 @@
|
||||
"""Push-level unseen evaluation, explicit coverage and paired adoption evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
from openultrasast.push.report import CAPABILITY_VETOES
|
||||
|
||||
STATES = ("instrument_failure", "unanalysable", "quick_only", "engine_covered")
|
||||
|
||||
|
||||
def coverage(record):
|
||||
instrument = record.get("instrument", {})
|
||||
push = record.get("push")
|
||||
if (
|
||||
not isinstance(push, dict)
|
||||
or not instrument.get("head_verified")
|
||||
or not instrument.get("base_verified")
|
||||
or instrument.get("suspect_fast")
|
||||
or instrument.get("failure")
|
||||
or instrument.get("hook_exit") not in (0, 1)
|
||||
or sum(instrument.get("changed_bytes", {}).values()) <= 0
|
||||
):
|
||||
return "instrument_failure"
|
||||
if "language_not_covered" in json.dumps([push.get("skipped", []), push.get("admission", {}).get("coverage_reasons", [])]) and not any(
|
||||
tier.get("bytes_read", 0) for tier in push.get("quick_tier", [])
|
||||
):
|
||||
return "unanalysable"
|
||||
if instrument.get("hook_bytes_read", 0) <= 0:
|
||||
return "instrument_failure"
|
||||
changed = set(instrument["changed_bytes"])
|
||||
if any(
|
||||
outcome.get("status") == "completed" and outcome.get("identity", {}).get("path") in changed
|
||||
for scan in push.get("scans", [])
|
||||
if scan.get("side") == "head"
|
||||
for outcome in scan.get("scan", {}).get("question_outcomes", [])
|
||||
):
|
||||
return "engine_covered"
|
||||
return "quick_only"
|
||||
|
||||
|
||||
def findings(record):
|
||||
push = record.get("push", {})
|
||||
admission = push.get("admission", {})
|
||||
for defect in admission.get("defects", []):
|
||||
for path, line in defect.get("locations", []):
|
||||
yield "alerts", {**defect, "path": path, "line": line}
|
||||
seen = set()
|
||||
for disposition in admission.get("dispositions", []):
|
||||
delta = disposition.get("candidate", {}).get("delta", {})
|
||||
operation = delta.get("head_operation")
|
||||
if (
|
||||
disposition.get("admitted")
|
||||
or delta.get("novelty") not in {"new", "worsened"}
|
||||
or not set(disposition.get("reasons", [])) <= CAPABILITY_VETOES
|
||||
or operation is None
|
||||
or delta.get("witness") is None
|
||||
or delta.get("defect_id") in seen
|
||||
):
|
||||
continue
|
||||
seen.add(delta.get("defect_id"))
|
||||
yield "advisory", {**operation, "family": delta.get("family")}
|
||||
taxonomy = load_families()
|
||||
for tier in push.get("quick_tier", []):
|
||||
for finding in tier.get("findings", []):
|
||||
family = taxonomy.family_of_cwe(finding.get("cwe") or "")
|
||||
yield "quick", {**finding, "family": family.id if family else None}
|
||||
|
||||
|
||||
def location_match(record, finding):
|
||||
for file in record.get("files", []):
|
||||
if finding.get("path") != file["path"]:
|
||||
continue
|
||||
line = finding.get("line", 0)
|
||||
if any(start - 10 <= line <= end + 10 for start, end in file.get("head_lines", [])):
|
||||
return True
|
||||
if record.get("family") != "config_secrets" and file["path"] == record.get("path", file["path"]):
|
||||
function = record.get("function")
|
||||
if function and function != "<global>" and finding.get("function") == function:
|
||||
return True
|
||||
span = record.get("function_lines") or file.get("function_lines")
|
||||
if span and span[0] <= line <= span[1]:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def catches(record):
|
||||
return sorted(
|
||||
{
|
||||
stream
|
||||
for stream, finding in findings(record)
|
||||
if finding.get("family") == record.get("family") and location_match(record, finding)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def false_alarm(record, *, alerts_only=False):
|
||||
return any(not alerts_only or stream == "alerts" for stream, _ in findings(record))
|
||||
|
||||
|
||||
def wilson(successes, n):
|
||||
if n == 0:
|
||||
return [None, None]
|
||||
p, z = successes / n, 1.96
|
||||
centre = (p + z * z / (2 * n)) / (1 + z * z / n)
|
||||
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
|
||||
return [max(0, centre - half), min(1, centre + half)]
|
||||
|
||||
|
||||
def required_n(p, width):
|
||||
for n in range(1, 1000000):
|
||||
low, high = wilson(p * n, n)
|
||||
if max(p - low, high - p) <= width:
|
||||
return n
|
||||
raise ValueError("requested precision exceeds sizing range")
|
||||
|
||||
|
||||
def rate(values):
|
||||
n, k = len(values), sum(values)
|
||||
return {"n": n, "count": k, "rate": k / n if n else None, "wilson95": wilson(k, n)}
|
||||
|
||||
|
||||
def cluster_interval(rows, *, seed=0, draws=2000):
|
||||
clusters = defaultdict(list)
|
||||
for repo, value in rows:
|
||||
clusters[repo].append(value)
|
||||
if not clusters:
|
||||
return [None, None]
|
||||
rng = random.Random(seed)
|
||||
groups = list(clusters.values())
|
||||
samples = []
|
||||
for _ in range(draws):
|
||||
values = [value for group in rng.choices(groups, k=len(groups)) for value in group]
|
||||
samples.append(sum(values) / len(values))
|
||||
samples.sort()
|
||||
return [samples[int(0.025 * (draws - 1))], samples[int(0.975 * (draws - 1))]]
|
||||
|
||||
|
||||
def icc(rows):
|
||||
clusters = defaultdict(list)
|
||||
for repo, value in rows:
|
||||
clusters[repo].append(value)
|
||||
n, k = len(rows), len(clusters)
|
||||
if k < 2 or n <= k:
|
||||
return None
|
||||
mean = sum(v for _, v in rows) / n
|
||||
between = sum(len(v) * (sum(v) / len(v) - mean) ** 2 for v in clusters.values()) / (k - 1)
|
||||
within = sum(sum((x - sum(v) / len(v)) ** 2 for x in v) for v in clusters.values()) / (n - k)
|
||||
size = (n - sum(len(v) ** 2 for v in clusters.values()) / n) / (k - 1)
|
||||
denominator = between + (size - 1) * within
|
||||
return (between - within) / denominator if denominator else 0.0
|
||||
|
||||
|
||||
def mcnemar(a, b):
|
||||
pairs = list(zip(a, b, strict=True))
|
||||
losses = sum(x and not y for x, y in pairs)
|
||||
gains = sum(y and not x for x, y in pairs)
|
||||
n = losses + gains
|
||||
return min(1.0, 2 * sum(math.comb(n, i) for i in range(min(losses, gains) + 1)) / 2**n) if n else 1.0
|
||||
|
||||
|
||||
def score(records, *, seed=0):
|
||||
counts = Counter(coverage(r) for r in records)
|
||||
if not records or counts["instrument_failure"] / len(records) > 0.05:
|
||||
raise ValueError("slice exceeds 5% instrument_failure (or is empty)")
|
||||
usable = [r for r in records if coverage(r) != "instrument_failure"]
|
||||
|
||||
def metrics(rows):
|
||||
ordinary = [r for r in rows if r["kind"] == "ordinary"]
|
||||
vulnerable = [r for r in rows if r["kind"] != "ordinary"]
|
||||
clustered = [(r["repo"], int(false_alarm(r))) for r in ordinary]
|
||||
return {
|
||||
"catch": rate([bool(catches(r)) for r in vulnerable]),
|
||||
"false_alarm": {
|
||||
**rate([false_alarm(r) for r in ordinary]),
|
||||
"cluster95": cluster_interval(clustered, seed=seed),
|
||||
"icc": icc(clustered),
|
||||
},
|
||||
"alerts_only": rate([false_alarm(r, alerts_only=True) for r in ordinary]),
|
||||
}
|
||||
|
||||
breakdown = {}
|
||||
for key in ("family", "method", "post_cutoff"):
|
||||
groups = defaultdict(list)
|
||||
for r in usable:
|
||||
groups[str(r.get(key, "unknown"))].append(r)
|
||||
breakdown[key] = {key: metrics(rows) for key, rows in sorted(groups.items())}
|
||||
breakdown["coverage"] = {state: metrics([r for r in usable if coverage(r) == state]) for state in STATES[1:]}
|
||||
breakdown["stream"] = {
|
||||
stream: rate([stream in catches(r) for r in usable if r["kind"] != "ordinary"]) for stream in ("alerts", "advisory", "quick")
|
||||
}
|
||||
return {
|
||||
**metrics(usable),
|
||||
"total": len(records),
|
||||
"deadlines": [
|
||||
{"lane": lane, "deadline_seconds": deadline, "deadline_scale": factor, "applied_deadline_scale": applied, "count": count}
|
||||
for (lane, deadline, factor, applied), count in sorted(
|
||||
Counter(
|
||||
(r.get("lane", "unknown"), r.get("deadline_seconds"), r.get("deadline_scale"), r.get("applied_deadline_scale"))
|
||||
for r in records
|
||||
).items(),
|
||||
key=lambda pair: str(pair[0]),
|
||||
)
|
||||
],
|
||||
"coverage": {state: counts[state] for state in STATES},
|
||||
"breakdowns": breakdown,
|
||||
"confirmed": metrics([r for r in usable if r.get("label_check", r.get("confirmed")) in ("confirmed", True)]),
|
||||
}
|
||||
|
||||
|
||||
def agreement(cluster, native):
|
||||
"""Missing, failed or wrong-lane records are unknown, never matching quiet pushes."""
|
||||
cc = {r["id"]: r for r in cluster}
|
||||
if len(cc) != len(cluster) or len({r["id"] for r in native}) != len(native):
|
||||
raise ValueError("duplicate agreement identities")
|
||||
pairs, unknown, disagreements = [], [], []
|
||||
for n in sorted(native, key=lambda r: r["id"]):
|
||||
c = cc.get(n["id"])
|
||||
if (
|
||||
c is None
|
||||
or c.get("lane") not in {"ax", "kind"}
|
||||
or n.get("lane") != "docker"
|
||||
or c.get("image") != n.get("image")
|
||||
or c.get("base_deadline_seconds") != n.get("base_deadline_seconds")
|
||||
or coverage(c) == "instrument_failure"
|
||||
or coverage(n) == "instrument_failure"
|
||||
):
|
||||
unknown.append(n["id"])
|
||||
continue
|
||||
cluster_flag, native_flag = false_alarm(c), false_alarm(n)
|
||||
same = cluster_flag == native_flag
|
||||
pairs.append({"change_id": n["id"], "cluster_flagged": cluster_flag, "native_flagged": native_flag, "same": same})
|
||||
if not same:
|
||||
disagreements.append(n["id"])
|
||||
report = rate([p["same"] for p in pairs])
|
||||
eligible = bool(pairs) and not unknown and report["wilson95"][0] >= 0.95
|
||||
return {
|
||||
**report,
|
||||
"requested": len(native),
|
||||
"pairs": pairs,
|
||||
"unknown": unknown,
|
||||
"disagreements": disagreements,
|
||||
"baseline_eligible": eligible,
|
||||
"flagged": not eligible,
|
||||
}
|
||||
|
||||
|
||||
def budgets(rows, *, thresholds=None, binary=False):
|
||||
ordinary = [r for r in rows if r["kind"] == "ordinary"]
|
||||
vulnerable = [r for r in rows if r["kind"] != "ordinary"]
|
||||
if binary:
|
||||
fa = rate([r["prediction"] for r in ordinary])
|
||||
return {
|
||||
"catch": rate([r["prediction"] for r in vulnerable]),
|
||||
"false_alarm": fa,
|
||||
"budgets_met": [f"{budget}%" for budget in (1, 5, 10) if fa["rate"] is not None and fa["rate"] <= budget / 100],
|
||||
}
|
||||
output = {}
|
||||
for budget in (1, 5, 10):
|
||||
key = f"{budget}%"
|
||||
if thresholds is not None and key not in thresholds:
|
||||
continue
|
||||
if not ordinary:
|
||||
output[key] = {"label": "not_run"}
|
||||
continue
|
||||
if thresholds is not None:
|
||||
threshold = thresholds[key]
|
||||
else:
|
||||
candidates = sorted({r["score"] for r in rows})
|
||||
candidates.append(math.nextafter(candidates[-1], math.inf))
|
||||
threshold = next(t for t in candidates if sum(r["score"] >= t for r in ordinary) / len(ordinary) <= budget / 100)
|
||||
output[key] = {
|
||||
"threshold": threshold,
|
||||
"label": "fixed" if thresholds is not None else "exploratory",
|
||||
"catch": rate([r["score"] >= threshold for r in vulnerable]),
|
||||
"false_alarm": rate([r["score"] >= threshold for r in ordinary]),
|
||||
}
|
||||
return output
|
||||
|
||||
|
||||
def gate(a, b, *, slice_id=None, informed=False, binary=True, threshold=None, threshold_fixed=False, seed=0):
|
||||
not_run = {"gate": "not_run"}
|
||||
if not slice_id or informed or not a or not b or (not binary and (not threshold_fixed or threshold is None)):
|
||||
return not_run
|
||||
try:
|
||||
score(a, seed=seed)
|
||||
score(b, seed=seed)
|
||||
except ValueError:
|
||||
return not_run
|
||||
aa, bb = {r["id"]: r for r in a}, {r["id"]: r for r in b}
|
||||
if len(aa) != len(a) or len(bb) != len(b) or aa.keys() != bb.keys():
|
||||
return not_run
|
||||
pairs = [
|
||||
(aa[key], bb[key]) for key in sorted(aa) if coverage(aa[key]) != "instrument_failure" and coverage(bb[key]) != "instrument_failure"
|
||||
]
|
||||
if any(x["kind"] != y["kind"] or x["repo"] != y["repo"] for x, y in pairs):
|
||||
return not_run
|
||||
vulnerable = [(x, y) for x, y in pairs if x["kind"] != "ordinary"]
|
||||
ordinary = [(x, y) for x, y in pairs if x["kind"] == "ordinary"]
|
||||
if not vulnerable or not ordinary:
|
||||
return not_run
|
||||
ca = [bool(catches(x)) for x, _ in vulnerable]
|
||||
cb = [bool(catches(y)) if binary else y["score"] >= threshold for _, y in vulnerable]
|
||||
fa = [false_alarm(x) for x, _ in ordinary]
|
||||
fb = [false_alarm(y) if binary else y["score"] >= threshold for _, y in ordinary]
|
||||
catch_p, fa_p = mcnemar(ca, cb), mcnemar(fa, fb)
|
||||
regression = (sum(cb) < sum(ca) and catch_p < 0.05) or (binary and sum(fb) > sum(fa) and fa_p < 0.05)
|
||||
return {
|
||||
"gate": "regression" if regression else "pass",
|
||||
"slice": slice_id,
|
||||
"catch": {"difference": sum(cb) / len(cb) - sum(ca) / len(ca), "mcnemar_p": catch_p},
|
||||
"false_alarm": {
|
||||
"difference": sum(fb) / len(fb) - sum(fa) / len(fa),
|
||||
"mcnemar_p": fa_p,
|
||||
"cluster95": cluster_interval([(x["repo"], int(v) - int(u)) for (x, _), u, v in zip(ordinary, fa, fb, strict=True)], seed=seed),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_records(path):
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text())
|
||||
rows = []
|
||||
for entry in json.loads((path / "inputs.json").read_text()):
|
||||
result = path / (hashlib.sha256(entry["item_id"].encode()).hexdigest() + ".json")
|
||||
record = json.loads(result.read_text()) if result.exists() else {}
|
||||
if record.get("input_digest") != entry["input_digest"]:
|
||||
record = {}
|
||||
rows.append({**entry["change"], **record, "id": entry["change"]["id"]})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("records", type=Path, help="JSON array of joined change/instrument/push records")
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--against", type=Path, help="production-arm records for a paired adoption gate")
|
||||
parser.add_argument("--native", type=Path, help="native sample root; auto-detected under replay root")
|
||||
parser.add_argument("--slice-id")
|
||||
parser.add_argument("--informed", action="store_true")
|
||||
args = parser.parse_args()
|
||||
current = load_records(args.records)
|
||||
result = score(current)
|
||||
native = args.native or (args.records / "native" if args.records.is_dir() and (args.records / "native").exists() else None)
|
||||
if native:
|
||||
result["native_agreement"] = agreement(current, load_records(native))
|
||||
result["baseline_eligible"] = result["native_agreement"]["baseline_eligible"]
|
||||
result["flagged"] = result["native_agreement"]["flagged"]
|
||||
result["unseen"] = gate(load_records(args.against) if args.against else [], current, slice_id=args.slice_id, informed=args.informed)
|
||||
if native and result["flagged"]:
|
||||
result["unseen"] = {"gate": "not_run", "reason": "native_agreement"}
|
||||
args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,331 +0,0 @@
|
||||
"""Advisory selection. Network adapters are constructed only by the coordinator CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
|
||||
from .eligibility import RepositoryNotFound, normalize
|
||||
|
||||
ECOSYSTEMS = ("pip", "npm", "maven", "composer")
|
||||
OSV_ECOSYSTEMS = dict(zip(ECOSYSTEMS, ("PyPI", "npm", "Maven", "Packagist"), strict=True))
|
||||
FAMILIES = {"injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "config_secrets"}
|
||||
PERMISSIVE = {"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "0BSD", "Unlicense", "Zlib", "PostgreSQL", "BSL-1.0"}
|
||||
FIX = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/commit/([a-fA-F0-9]{7,40})(?:[/?#].*)?$", re.I)
|
||||
|
||||
|
||||
class Rejected(ValueError):
|
||||
"""Expected eligibility rejection; messages are fixed reason codes, never names."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repository: str
|
||||
url: str
|
||||
fix: str
|
||||
parent: str
|
||||
advisory: str
|
||||
ecosystem: str
|
||||
family: str
|
||||
post_cutoff: bool
|
||||
first_affected: str | None
|
||||
license_spdx: str
|
||||
license_class: str
|
||||
license_ref: str
|
||||
license_path: str
|
||||
|
||||
|
||||
def post_cutoff(row: dict) -> bool:
|
||||
return date.fromisoformat(row["published_at"][:10]) > date(2026, 6, 1)
|
||||
|
||||
|
||||
def fix_links(row: dict) -> set[tuple[str, str]]:
|
||||
links = set()
|
||||
for reference in row.get("references", []):
|
||||
url = reference if isinstance(reference, str) else reference.get("url", "")
|
||||
match = FIX.fullmatch(url)
|
||||
if match:
|
||||
links.add((normalize(match[1]), match[2].lower()))
|
||||
return links
|
||||
|
||||
|
||||
def ecosystem(row: dict) -> str | None:
|
||||
values = {v["package"]["ecosystem"].lower() for v in row.get("vulnerabilities", [])}
|
||||
return next((e for e in ECOSYSTEMS if e in values), None)
|
||||
|
||||
|
||||
def reduce_advisory(row: dict) -> dict:
|
||||
"""Keep only selection inputs, including every fix needed by the exclusion index.
|
||||
|
||||
Preserve the parser's shape so fresh and resumed pages follow the same path.
|
||||
Commit parents, licenses and OSV brackets are resolved later into Candidate.
|
||||
"""
|
||||
return {
|
||||
"ghsa_id": row["ghsa_id"],
|
||||
"published_at": row["published_at"],
|
||||
"type": row.get("type", "reviewed"),
|
||||
"withdrawn_at": row.get("withdrawn_at"),
|
||||
"cwes": [{"cwe_id": c["cwe_id"]} for c in row.get("cwes", [])],
|
||||
"vulnerabilities": [{"package": {key: v["package"][key] for key in ("ecosystem", "name")}} for v in row.get("vulnerabilities", [])],
|
||||
"references": [f"https://github.com/{name}/commit/{sha}" for name, sha in sorted(fix_links(row))],
|
||||
}
|
||||
|
||||
|
||||
def ordered_advisories(rows: list[dict], seed: int) -> list[dict]:
|
||||
"""Stable seeded ordering, post-cutoff first in each ecosystem; round-robin strata."""
|
||||
rng = random.Random(seed)
|
||||
groups = []
|
||||
for eco in ECOSYSTEMS:
|
||||
group = sorted((r for r in rows if ecosystem(r) == eco), key=lambda r: r["ghsa_id"])
|
||||
rng.shuffle(group)
|
||||
group.sort(key=lambda r: not post_cutoff(r))
|
||||
groups.append(group)
|
||||
return [group[i] for i in range(max(map(len, groups), default=0)) for group in groups if i < len(group)]
|
||||
|
||||
|
||||
def first_affected(record: dict, row: dict, eco: str) -> str | None:
|
||||
packages = {v["package"]["name"] for v in row["vulnerabilities"] if v["package"]["ecosystem"].lower() == eco}
|
||||
versions = set()
|
||||
for affected in record.get("affected", []):
|
||||
package = affected.get("package", {})
|
||||
if package.get("ecosystem") != OSV_ECOSYSTEMS[eco] or package.get("name") not in packages:
|
||||
continue
|
||||
for span in affected.get("ranges", []):
|
||||
if span.get("type") not in {"ECOSYSTEM", "SEMVER"}:
|
||||
continue
|
||||
for event in span.get("events", []):
|
||||
if event.get("introduced") not in {None, "0"}:
|
||||
versions.add(event["introduced"])
|
||||
if len(versions) > 1:
|
||||
raise Rejected("ambiguous_version_bracket")
|
||||
return next(iter(versions), None)
|
||||
|
||||
|
||||
def license_info(record: dict) -> tuple[str, str]:
|
||||
# Explicit pinned-file SPDX wins. For GitHub-recognized full texts its license endpoint
|
||||
# provides the detected SPDX. Unknown/custom/compound declarations stay private.
|
||||
declared = re.search(r"SPDX-License-Identifier:\s*([^\r\n]+)", record.get("text", ""))
|
||||
spdx = declared[1].strip().removesuffix("*/").strip() if declared else record.get("spdx_id") or "NONE"
|
||||
return spdx, "permissive" if spdx in PERMISSIVE else "private"
|
||||
|
||||
|
||||
def candidate(row: dict, github, osv) -> Candidate:
|
||||
if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
|
||||
raise Rejected("date")
|
||||
if row.get("type", "reviewed") != "reviewed" or row.get("withdrawn_at"):
|
||||
raise Rejected("advisory_status")
|
||||
eco = ecosystem(row)
|
||||
if eco is None:
|
||||
raise Rejected("ecosystem")
|
||||
taxonomy = load_families()
|
||||
families = {f.id for c in row.get("cwes", []) if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in FAMILIES}
|
||||
if len(families) != 1:
|
||||
raise Rejected("family")
|
||||
links = fix_links(row)
|
||||
if len(links) != 1:
|
||||
raise Rejected("fix_links")
|
||||
name, fix = next(iter(links))
|
||||
repo = github.get_repo(name)
|
||||
url = urlsplit(repo.get("clone_url", ""))
|
||||
if (
|
||||
repo.get("private", True)
|
||||
or not 0 <= repo.get("size", -1) <= 500 * 1024
|
||||
or url.scheme != "https"
|
||||
or url.hostname != "github.com"
|
||||
or url.username
|
||||
or url.password
|
||||
or url.query
|
||||
or url.fragment
|
||||
or normalize(repo["clone_url"]) != normalize(repo["full_name"])
|
||||
):
|
||||
raise Rejected("repository")
|
||||
commit = github.get_commit(repo["full_name"], fix)
|
||||
if len(commit["parents"]) != 1:
|
||||
raise Rejected("parents")
|
||||
parent = commit["parents"][0]["sha"]
|
||||
license_record = github.get_license(repo["full_name"], parent)
|
||||
spdx, license_class = license_info(license_record)
|
||||
return Candidate(
|
||||
normalize(repo["full_name"]),
|
||||
repo["clone_url"],
|
||||
commit["sha"],
|
||||
parent,
|
||||
row["ghsa_id"],
|
||||
eco,
|
||||
next(iter(families)),
|
||||
post_cutoff(row),
|
||||
first_affected(osv.get(row["ghsa_id"]), row, eco),
|
||||
spdx,
|
||||
license_class,
|
||||
parent,
|
||||
license_record.get("path", ""),
|
||||
)
|
||||
|
||||
|
||||
class JSONTransport:
|
||||
"""Coordinator GET transport with bounded retries; all I/O and waits are injectable."""
|
||||
|
||||
def __init__(self, *, opener=None, sleeper=None, clock=None):
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
self.opener = opener or urlopen
|
||||
self.sleeper = sleeper or time.sleep
|
||||
self.clock = clock or time.time
|
||||
self.attempts = 0
|
||||
self.rate_limit = {}
|
||||
self.response_headers = {}
|
||||
|
||||
def get(self, url: str, headers: dict) -> object:
|
||||
from http.client import IncompleteRead
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request
|
||||
|
||||
for attempt in range(5):
|
||||
self.attempts += 1
|
||||
retry_headers, limited = {}, False
|
||||
try:
|
||||
with self.opener(Request(url, headers=headers), timeout=60) as response:
|
||||
retry_headers = getattr(response, "headers", {})
|
||||
self.record_rate(retry_headers)
|
||||
body = json.load(response)
|
||||
self.response_headers = retry_headers
|
||||
return body
|
||||
except HTTPError as exc:
|
||||
self.record_rate(exc.headers)
|
||||
if exc.code == 404:
|
||||
raise RepositoryNotFound("not_found") from None
|
||||
limited = exc.code == 429 or (
|
||||
exc.code == 403 and (exc.headers.get("X-RateLimit-Remaining") == "0" or exc.headers.get("Retry-After"))
|
||||
)
|
||||
transient = exc.code in {500, 502, 503, 504}
|
||||
if attempt == 4 or not (limited or transient):
|
||||
raise ValueError("http_failure") from None
|
||||
retry_headers = exc.headers
|
||||
except (URLError, TimeoutError, ConnectionError, IncompleteRead, json.JSONDecodeError):
|
||||
if attempt == 4:
|
||||
raise ValueError("http_failure") from None
|
||||
except Exception:
|
||||
raise ValueError("http_failure") from None
|
||||
try:
|
||||
delay = float(retry_headers.get("Retry-After", "0"))
|
||||
if limited and not delay:
|
||||
delay = max(60, float(retry_headers.get("X-RateLimit-Reset", "0")) - self.clock() + 1)
|
||||
delay = max(delay, 2**attempt)
|
||||
if delay > 3605:
|
||||
raise ValueError("retry_window")
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError("http_retry_failed") from None
|
||||
self.sleeper(delay)
|
||||
raise ValueError("http_failure")
|
||||
|
||||
def record_rate(self, headers):
|
||||
self.rate_limit = {
|
||||
key: headers.get(header)
|
||||
for key, header in (
|
||||
("remaining", "X-RateLimit-Remaining"),
|
||||
("limit", "X-RateLimit-Limit"),
|
||||
("reset", "X-RateLimit-Reset"),
|
||||
("retry_after", "Retry-After"),
|
||||
)
|
||||
if headers.get(header) is not None
|
||||
}
|
||||
|
||||
|
||||
class GitHub:
|
||||
def __init__(self, token: str, transport=None):
|
||||
if not token:
|
||||
raise ValueError("missing_gh_token")
|
||||
self.transport = transport or JSONTransport()
|
||||
self.headers = {"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json"}
|
||||
self.repos: dict[str, dict] = {}
|
||||
self.calls = 0
|
||||
|
||||
def get(self, path: str):
|
||||
self.calls += 1
|
||||
url = path if path.startswith("https://api.github.com/") else "https://api.github.com" + path
|
||||
return self.transport.get(url, self.headers)
|
||||
|
||||
def get_repo(self, name: str) -> dict:
|
||||
name = normalize(name)
|
||||
if name not in self.repos:
|
||||
self.repos[name] = self.get("/repos/" + name)
|
||||
return self.repos[name]
|
||||
|
||||
def get_commit(self, name: str, sha: str) -> dict:
|
||||
return self.get("/repos/" + normalize(name) + "/commits/" + sha)
|
||||
|
||||
def get_license(self, name: str, ref: str) -> dict:
|
||||
try:
|
||||
row = self.get("/repos/" + normalize(name) + "/license?" + urlencode({"ref": ref}))
|
||||
except RepositoryNotFound:
|
||||
return {"spdx_id": "NONE", "text": "", "path": ""}
|
||||
return {
|
||||
"spdx_id": (row.get("license") or {}).get("spdx_id"),
|
||||
"path": row.get("path", ""),
|
||||
"text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
|
||||
}
|
||||
|
||||
def advisory_page(self, next_url: str | None = None, *, ecosystem: str) -> dict:
|
||||
if ecosystem not in ECOSYSTEMS:
|
||||
raise ValueError("invalid_ecosystem")
|
||||
# Oldest first: newly published advisories do not shift already persisted pages.
|
||||
parameters = {
|
||||
"type": "reviewed",
|
||||
"per_page": 100,
|
||||
"sort": "published",
|
||||
"direction": "asc",
|
||||
"ecosystem": ecosystem,
|
||||
# Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md.
|
||||
"published": ">=2019-01-01",
|
||||
}
|
||||
taxonomy = load_families()
|
||||
cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes}
|
||||
# family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens,
|
||||
# fetch broadly rather than risk excluding a client-selectable advisory.
|
||||
# GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89".
|
||||
if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes):
|
||||
parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int))
|
||||
rows = self.get(next_url or "/advisories?" + urlencode(parameters))
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("invalid_advisory_page")
|
||||
link = self.transport.response_headers.get("Link", "")
|
||||
match = re.search(r'<([^>]+)>;\s*rel="next"', link)
|
||||
return {"rows": rows, "next_url": match[1] if match else None}
|
||||
|
||||
def advisories(self):
|
||||
"""Enumerate the candidate population, deduplicating multi-ecosystem advisories."""
|
||||
seen = set()
|
||||
for ecosystem in ECOSYSTEMS:
|
||||
next_url = None
|
||||
while True:
|
||||
result = self.advisory_page(next_url, ecosystem=ecosystem)
|
||||
for row in result["rows"]:
|
||||
if row["ghsa_id"] not in seen:
|
||||
seen.add(row["ghsa_id"])
|
||||
yield row
|
||||
next_url = result["next_url"]
|
||||
if next_url is None:
|
||||
break
|
||||
|
||||
|
||||
class OSV:
|
||||
def __init__(self, transport=None):
|
||||
self.transport = transport or JSONTransport()
|
||||
self.calls = 0
|
||||
|
||||
def get(self, identifier: str) -> dict:
|
||||
if not re.fullmatch(r"GHSA-[\w-]+", identifier):
|
||||
raise ValueError("invalid_advisory_id")
|
||||
self.calls += 1
|
||||
try:
|
||||
return self.transport.get("https://api.osv.dev/v1/vulns/" + identifier, {})
|
||||
except RepositoryNotFound:
|
||||
return {} # No OSV bracket is distinct from a failed service call.
|
||||
+3
-18
@@ -17,21 +17,6 @@ This page uses two words strictly:
|
||||
been made.** Section B is guidance, not a record. (kind runs a Kubernetes cluster inside Docker
|
||||
containers on one machine.)
|
||||
|
||||
The kind profile runs on a 7.7 GB server VM. Observability is off by default
|
||||
(`kind_observability = false` in `ops/k8s/profiles/kind.toml`). This saves the measured
|
||||
930 MiB used by `otel-system`, about 40% of the cluster's 2.3 GB. The smoke Run still
|
||||
completes with those deployments stopped. To enable the stack, run
|
||||
`OUSAST_KIND_OBSERVABILITY=1 ops/ax/up.sh` or set `kind_observability = true` in the profile.
|
||||
|
||||
`up.sh` stages copied Substrate manifests in a temporary checkout view and installs through
|
||||
`hack/install-ate-kind.sh`. The local `ops/ax/substrate-kind-lean/` overlay removes the collector,
|
||||
Prometheus, Jaeger and their supporting resources before installation. This preserves the
|
||||
installer's secret, CRD and image setup. The upstream checkout is unchanged. The shared
|
||||
`ate-otel-config` disables trace, metric and log export and the OTEL SDK; it removes the old
|
||||
collector endpoint and metric timing keys. On an existing cluster, `up.sh` scales the three
|
||||
observability deployments to zero, replaces the ConfigMap data and rolls its consumers when
|
||||
the configuration changes. Opting in restores the upstream ConfigMap and observability manifests.
|
||||
|
||||
## What runs where today (implemented)
|
||||
|
||||
```mermaid
|
||||
@@ -372,7 +357,7 @@ This section answers the maintainer's questions from the code and the upstream c
|
||||
|
||||
The decisions of 2026-10-02 set the target: **two profiles of one code base**.
|
||||
|
||||
- `kind` is this server VM. It stays a first-class local execution mode. Every change is proven here first.
|
||||
- `kind` is this laptop. It stays a first-class local execution mode. Every change is proven here first.
|
||||
- `k3s` is production. (k3s is a small Kubernetes distribution.)
|
||||
|
||||
Everything marked **planned** is not built. The rest is what runs on this host today.
|
||||
@@ -383,7 +368,7 @@ flowchart LR
|
||||
hook["ousast pre-push: local, 30 s deadline"]
|
||||
cli["ousast plane run, remote mode (planned)"]
|
||||
end
|
||||
subgraph k3s ["k3s cluster (the kind profile is the same picture on one server VM)"]
|
||||
subgraph k3s ["k3s cluster (the kind profile is the same picture on one laptop)"]
|
||||
subgraph axns ["Namespace ax-system"]
|
||||
axs["ax-server Deployment + Redis"]
|
||||
end
|
||||
@@ -418,7 +403,7 @@ This picture has no inbound path:
|
||||
- the tasks push their artifacts to the store through the egress gateway;
|
||||
- the CLI reads the store.
|
||||
|
||||
Today's EndpointSlice back to the server VM (`ops/ax/receiver-service.yaml.tmpl`) belongs to the kind
|
||||
Today's EndpointSlice back to a laptop (`ops/ax/receiver-service.yaml.tmpl`) belongs to the kind
|
||||
profile only. It does not appear in production.
|
||||
|
||||
### ax, Kubernetes and Agent Substrate in three sentences
|
||||
|
||||
+1
-1
@@ -159,7 +159,7 @@ flowchart LR
|
||||
ax is a Kubernetes application over Agent Substrate. Every Task runs the one runner image as an actor. Each actor
|
||||
occupies a whole worker. This code base targets two profiles:
|
||||
|
||||
- `kind` on a server VM (today).
|
||||
- `kind` on a laptop (today).
|
||||
- `k3s` in production (planned). There the same CLI submits Runs remotely, and the tasks deliver to the S3 store.
|
||||
|
||||
[Production topology: ax on a Kubernetes cluster](deployment.md#production-topology-ax-on-a-kubernetes-cluster)
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
# Pre-push security for developers
|
||||
|
||||
You push code. This checks it for security regressions first. You do not annotate
|
||||
anything, you do not need a security background, and you do not change how you work.
|
||||
|
||||
## The short version
|
||||
|
||||
Install a git pre-push hook once. On every push it reads the code you changed,
|
||||
compares it against the known risks of the frameworks you already use, and warns
|
||||
you before the push leaves your machine. Where it can build and run your app, it
|
||||
goes further and proves an issue with a demonstration: a check that triggers the
|
||||
problem on your change and stops triggering once the problem is fixed. By default
|
||||
it only advises; it never blocks until you turn blocking on.
|
||||
|
||||
## Why this matters now
|
||||
|
||||
Your backlog is full and your schedule is not yours. Security findings compete
|
||||
with feature tickets, and a vague "possible vulnerability" loses that fight every
|
||||
time: no one gets capacity to chase a maybe. Meanwhile you and your coding agents
|
||||
ship quickly, and an agent writing code does not pause to audit itself. A
|
||||
pre-push check is the place to catch a regression that you, or an agent acting
|
||||
for you, introduced, before it reaches the remote. In an agent-driven workflow it
|
||||
is the oversight step: the gate that reviews what the agent wrote, every push,
|
||||
without you having to become the security reviewer.
|
||||
|
||||
## From a finding to capacity
|
||||
|
||||
The point of the proof is the prioritization conversation, not just your terminal.
|
||||
|
||||
A theoretical warning dies in the backlog. A demonstrated proof does not: it is a
|
||||
reproducible check that triggers the problem on the current code and stops once
|
||||
the code is fixed. That is the thing you take to a product owner or to the
|
||||
business when you need capacity. It turns "security thinks this might be a
|
||||
problem" into "here is the exact input, the exact behavior, and the one-line
|
||||
difference that stops it," which is something a product owner can weigh against a
|
||||
feature with real information instead of fear.
|
||||
|
||||
So the output is built to drop into your ticket system as evidence: a location, a
|
||||
concrete repair direction, and, where the app is runnable, the reproduction
|
||||
itself. That is what gets a fix scheduled rather than deferred.
|
||||
|
||||
## What you do
|
||||
|
||||
1. Install the hook in a repo: `ousast pre-push install`.
|
||||
2. Push normally. Findings print at push time.
|
||||
3. Optionally set model keys and a spend cap in `.env`; nothing else is required.
|
||||
|
||||
No code annotations. No markup. No per-project security config.
|
||||
|
||||
## How it decides, without annotations
|
||||
|
||||
- **The tool brings the security knowledge.** It ships framework facts: the
|
||||
sources, sinks, sanitizers, and guards for common frameworks. Your code does
|
||||
not declare any of this.
|
||||
- **It looks only at your change.** It builds a graph of the changed code and
|
||||
checks it against those framework facts, scoped to the diff, not the whole repo.
|
||||
- **It can prove, not just warn.** When the app is runnable, the proof arm builds
|
||||
and runs it and demonstrates the issue by execution, so a finding is a
|
||||
reproducible fact, not a confidence score.
|
||||
|
||||
## What you control
|
||||
|
||||
- **Advisory or blocking.** Advisory by default; `OUSAST_PUSH_MODE=blocking`
|
||||
turns it into a gate. Adopt in advisory mode, build trust, then block.
|
||||
- **Budget and deadline.** One budget covers the whole push; if it runs out you
|
||||
get an honest "incomplete coverage" notice, never a made-up verdict.
|
||||
- **Where proof runs.** Local sandbox, a container, or a shared cluster.
|
||||
- **Models and spend.** Keys and caps live in `.env`.
|
||||
|
||||
## Honest limits
|
||||
|
||||
- **Coverage follows support.** It helps where the tool has framework facts and a
|
||||
language grammar; outside that it is weak, and third-party or vendor code is not
|
||||
analyzed.
|
||||
- **Large repositories can be slow.** On big codebases the deep analysis can
|
||||
exceed a comfortable push deadline today; that is an open engineering problem.
|
||||
- **Generalization is being measured, not asserted.** Whether it reliably catches
|
||||
real regressions and stays quiet on ordinary changes in repositories it has
|
||||
never seen is under active measurement. We consider the tool wrong if it misses
|
||||
real regressions or alarms on ordinary changes on unseen repositories, and that
|
||||
is exactly the catch-rate and false-alarm number the unseen-repo evaluation is
|
||||
producing now. Treat the advisory output accordingly until that number exists.
|
||||
|
||||
## Where this is going
|
||||
|
||||
An MCP agent and skills so your editor or your coding agent gets the same check
|
||||
earlier, inline, before the push, so feedback arrives while you are still writing
|
||||
rather than at the gate. See the follow-up spec `mcp-shift-left-agent`.
|
||||
@@ -214,42 +214,3 @@ bucket `OUSAST_MEMORY_TEST_BUCKET`, else `S3_BUCKET`. The tests:
|
||||
- never configure the bucket.
|
||||
|
||||
More on the host setup: [ax on this host](ops/ax/README.md).
|
||||
|
||||
## Scoped account for engine workers
|
||||
|
||||
Create a **new account** for the k3s engine Deployment. Attach only this policy.
|
||||
Do not give it the existing memory-store account's policies.
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "ListEnginePrefixes",
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:ListBucket",
|
||||
"Resource": "arn:aws:s3:::sast-memory",
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": ["engine-queue/", "engine-queue/*", "engine-results/", "engine-results/*"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "EngineObjectsOnly",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::sast-memory/engine-queue/*",
|
||||
"arn:aws:s3:::sast-memory/engine-results/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The cluster can read and write only queued engine inputs, claims, and results.
|
||||
It cannot list or read the rest of the memory store. It cannot change bucket
|
||||
settings. The host queue dispatcher can use the same scoped account. Store its
|
||||
keys in the `ousast-engine-s3` Secret; no key values belong in manifests.
|
||||
Deployment and coordinator commands are in `ops/k8s/engine/README.md`.
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# Use case: catch, prove, fix, re-verify
|
||||
|
||||
The tool closes the loop on a security regression. It finds the problem in your
|
||||
change, proves it with a reproducible check, and after the fix re-runs that same
|
||||
check to confirm the problem is gone. The proof becomes a regression test. It
|
||||
does not write the patch for you; it hands you a failing check and a repair
|
||||
direction, and your fix is confirmed when the check turns green.
|
||||
|
||||
## The loop
|
||||
|
||||
1. **Catch.** On the changed code, the static arm flags a likely regression with
|
||||
a location and a repair direction.
|
||||
2. **Prove.** Where the app is runnable, the proof arm builds and runs it and
|
||||
demonstrates the issue: a check that triggers the effect on the current code.
|
||||
3. **Fix.** You, or a coding agent, write the fix against the repair direction
|
||||
and the failing check. The tool does not generate the patch.
|
||||
4. **Re-verify absence.** The same proof is re-run against the fixed code. It no
|
||||
longer triggers. That is the bug's absence demonstrated by execution, not a
|
||||
claim that it "looks fixed."
|
||||
5. **Keep it.** The proof is a regression test. Commit it and the fix stays fixed.
|
||||
|
||||
## What "re-verify absence" means, precisely
|
||||
|
||||
A demonstrated finding is a differential: the proof triggers the effect on the
|
||||
vulnerable code and does not on the fixed code. Re-verifying a fix is the same
|
||||
operation run against your new revision. No effect means the specific issue the
|
||||
proof captured is gone. It does not claim the code is secure in general; it
|
||||
confirms that this proven issue no longer reproduces. That scoping is the point:
|
||||
it is honest about exactly what it checked.
|
||||
|
||||
## What it does and does not do
|
||||
|
||||
- It produces a repair direction and a failing proof, not an auto-generated patch.
|
||||
- It re-verifies the fix by re-running the proof; the verifier already runs a
|
||||
demo against a given revision, so this is the existing mechanism, not a new one.
|
||||
- Families without a runnable proof (for example access control or configuration)
|
||||
get the catch and the repair direction but no execution proof, and are advisory
|
||||
only.
|
||||
|
||||
## In CI, with GitHub Actions
|
||||
|
||||
`ousast pre-push . --base <rev> --head <rev> --deadline <seconds>` runs an
|
||||
explicit base-versus-head check, which is what an Action calls on a pull request.
|
||||
The example below is reference material, not an active workflow in this repo.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/pre-push-security.yml (example; adapt and enable in your repo)
|
||||
name: pre-push security
|
||||
on:
|
||||
pull_request:
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # base and head must both be present
|
||||
- name: Install OpenUltraSAST
|
||||
run: pipx install openultrasast # or your pinned distribution
|
||||
- name: Check the diff
|
||||
env:
|
||||
# model keys and spend caps, if the proof arm is used, come from secrets
|
||||
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
||||
run: |
|
||||
ousast pre-push . \
|
||||
--base "origin/${{ github.base_ref }}" \
|
||||
--head "${{ github.sha }}" \
|
||||
--deadline 300 \
|
||||
--mode advisory \
|
||||
--artifact "$GITHUB_WORKSPACE/pre-push.json"
|
||||
- name: Comment findings on the PR
|
||||
if: always()
|
||||
run: |
|
||||
# Placeholder: read pre-push.json and post the finding, its proof and the
|
||||
# repair direction as a PR comment. This formatter is not packaged yet
|
||||
# (see Honest status); wire your own, e.g. with gh pr comment.
|
||||
cat "$GITHUB_WORKSPACE/pre-push.json"
|
||||
```
|
||||
|
||||
The intended flow on a repository:
|
||||
|
||||
- A pull request scans base to head. A demonstrated finding is posted as a PR
|
||||
comment with its proof and repair direction, the evidence a developer takes to
|
||||
a product owner to win capacity.
|
||||
- The fix pull request re-runs the check. The proof flips from failing to
|
||||
passing, which is the fix confirmed.
|
||||
- The proof is committed as a regression test so the issue cannot return
|
||||
unnoticed.
|
||||
|
||||
## Honest status
|
||||
|
||||
Re-verify by execution and the `--base/--head` entry point exist today. The
|
||||
turnkey pieces around them are delivery work, not detection: the PR-comment
|
||||
formatter and the automatic landing of a proof as a committed regression test are
|
||||
not packaged yet, and the base-versus-head path inherits the open large-repo
|
||||
latency limit, so a long deadline or a cluster lane may be needed on big diffs.
|
||||
The full-repository CI scan (as opposed to a diff) is roadmap work
|
||||
(`plane-on-kubernetes`), not the pre-push diff path shown here.
|
||||
+1
-147
@@ -17,21 +17,6 @@ in `~/.cache/ousast/ax-src/`. The cluster's context, registry and pool sizes com
|
||||
(`ops/k8s/profiles/kind-images.json`), which the templates are re-pinned from and which is committed with them.
|
||||
`up.sh` and `smoke-run.sh` run the checkout's code with `OUSAST_PYTHON` (default `.venv/bin/python`).
|
||||
|
||||
The kind profile runs on a 7.7 GB server VM. Observability is off by default
|
||||
(`kind_observability = false` in `ops/k8s/profiles/kind.toml`). This saves the measured
|
||||
930 MiB used by `otel-system`, about 40% of the cluster's 2.3 GB. The smoke Run still
|
||||
completes with those deployments stopped. To enable the stack, run
|
||||
`OUSAST_KIND_OBSERVABILITY=1 ops/ax/up.sh` or set `kind_observability = true` in the profile.
|
||||
|
||||
`up.sh` stages copied Substrate manifests in a temporary checkout view and installs through
|
||||
`hack/install-ate-kind.sh`. The local `ops/ax/substrate-kind-lean/` overlay removes the collector,
|
||||
Prometheus, Jaeger and their supporting resources before installation. This preserves the
|
||||
installer's secret, CRD and image setup. The upstream checkout is unchanged. The shared
|
||||
`ate-otel-config` disables trace, metric and log export and the OTEL SDK; it removes the old
|
||||
collector endpoint and metric timing keys. On an existing cluster, `up.sh` scales the three
|
||||
observability deployments to zero, replaces the ConfigMap data and rolls its consumers when
|
||||
the configuration changes. Opting in restores the upstream ConfigMap and observability manifests.
|
||||
|
||||
## How a Run executes
|
||||
|
||||
A Run (`plane/runs/*.yaml`, kind `openultrasast.io/v1alpha1 Run`) is a DAG of steps; each step names an ax Task
|
||||
@@ -206,7 +191,7 @@ configure the bucket.
|
||||
No deployment outside this host's kind cluster has been made. The step-by-step guidance, with what is implemented
|
||||
and what is planned, is `docs/deployment.md` (the site page "Deployment"). The Model and Workspace manifests under
|
||||
`plane/` move as they are and the Task templates need their image reference re-pinned to your registry; these parts
|
||||
assume this server VM and must change first (checked 2026-10-02):
|
||||
assume this laptop and must change first (checked 2026-10-02):
|
||||
|
||||
| Assumption | Where | Needed in a real cluster |
|
||||
|---|---|---|
|
||||
@@ -228,134 +213,3 @@ pointing at your own bucket, a registry the workers can pull from, the runner im
|
||||
reachable as a Service, and the provider host allowed per task (which each task's EgressPolicy already does). The
|
||||
Model and Workspace manifests under `plane/` move unchanged; the Task templates get a new image reference; the kube
|
||||
context and `doctor`'s registry check are not configurable yet (the table above). Step by step: `docs/deployment.md`.
|
||||
|
||||
## Search executor and verifier task contract (operator verified 2026-10-05)
|
||||
|
||||
There are two workers and no queue. `no free workers available` is capacity
|
||||
back-pressure: the dispatcher retries the same apply with 15–30 seconds of jitter,
|
||||
up to the workload deadline. It does not retry a sandbox execution or route to the VM
|
||||
because of this response. A shared reservation limits all AX workloads in one host
|
||||
process to two tasks; separate host processes coordinate through AX's rejection.
|
||||
|
||||
Search executors build on the disk-backed `/workspace`, export a single archive, and
|
||||
are deleted before verifier repetitions start. The archive contains `checkout/`,
|
||||
`products/`, and `spec.json` with a `none` build recipe. A verifier receives only
|
||||
`VERIFY_INPUT_URL` and `RESULT_URL`; it never installs dependencies or fetches source.
|
||||
The executor's host-scoped `PREPARED_PUT_URL` carries the built output. The host checks
|
||||
its SHA-256 before passing it to fresh verifier tasks. Each side has three fresh runs.
|
||||
|
||||
Verifier egress to `files.because-security.com:443` is requested but **pending**.
|
||||
Until the operator enables it, live verifier input/output is blocked. The first input
|
||||
download tolerates up to 60 seconds of delayed policy activation.
|
||||
|
||||
`/` and `/tmp` consume the worker's 3 GiB RAM; `/workspace` has about 2 GiB usable disk.
|
||||
Entrypoints set `TMPDIR=/workspace/tmp`, `npm_config_cache=/workspace/.npm`,
|
||||
`MAVEN_OPTS=-Dmaven.repo.local=/workspace/.m2`, `COMPOSER_HOME=/workspace/.composer`,
|
||||
`PIP_CACHE_DIR=/workspace/.pip`, and `GRADLE_USER_HOME=/workspace/.gradle`.
|
||||
`OUSAST_SCRATCH_BYTES` overrides the default 2147483648-byte workspace guard. The guard
|
||||
includes caches and open output logs, checks running children and completed writes,
|
||||
and reports `could_not_build` / `scratch limit`. It is a monitored guard, not a hard
|
||||
filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM.
|
||||
|
||||
Repository clones must use HTTPS `git clone https://github.com/...git` or codeload
|
||||
archives, never `api.github.com`: tasks share one public IP. Search task inputs are
|
||||
presigned checkout archives, so these entrypoints do not clone repositories themselves.
|
||||
|
||||
## Image layers, publication and node cache budget
|
||||
|
||||
The task family has one shared runtime base, `ousast-task-base`, built from the
|
||||
public AX runner v0.3.1. `base-browser` adds Chromium to that exact published base
|
||||
(by digest); engine and replay do not carry Chromium. The VM/kind `openultrasast`
|
||||
image uses the same base and Joern recipe. Replay inherits the published engine
|
||||
digest in CI, so even a cold build cannot accidentally duplicate its Joern layers.
|
||||
|
||||
Java is **Temurin 21 JDK**, copied from `eclipse-temurin:21-jdk-jammy`.
|
||||
`ops/frontend-retention/install.py` explicitly requires Java 21 and invokes
|
||||
`java -cp ... dotty.tools.dotc.Main` using Joern's bundled Scala compiler.
|
||||
The shared JDK also provides `javac` for Maven/Gradle project builds.
|
||||
Curl/unzip and retention build scratch stay in a discarded Joern build stage.
|
||||
Joern stays above the base in the engine image (also used by replay and the VM
|
||||
lane); its compiler jars and retention provenance must not be blindly pruned.
|
||||
Runtime dependencies from `pyproject.toml` (core plus semantic/s3 for the VM lane)
|
||||
precede project code. Package installation uses `--no-deps --no-cache-dir`;
|
||||
its isolated packaging backend is temporary.
|
||||
|
||||
The base retains upstream Python 3.12 with venv/pip. Its explicit apt package list
|
||||
is `nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux
|
||||
git ca-certificates libstdc++6 zlib1g`. Composer is the trixie apt package.
|
||||
Chromium is installed only in `base-browser`. Apt caches/lists and docs/man/info
|
||||
are cleaned; dpkg excludes new docs/man/info. Files already in inherited AX
|
||||
layers still occupy space even if deleted.
|
||||
|
||||
Executor preparation supports pip, npm, Composer, Maven and Gradle builds.
|
||||
There is **no installed Gradle binary**: the executor invokes the selected project's
|
||||
`gradlew` through `/bin/sh`, including for wrappers without executable permissions.
|
||||
Gradle-wrapper distribution downloads and dependency downloads go through
|
||||
**public HTTPS from the executor**, subject to its egress policy and deadline.
|
||||
The verifier receives prepared products and performs no dependency downloads.
|
||||
|
||||
Rebuild policy:
|
||||
|
||||
- Dependency/toolchain changes: run **Task base image** (also triggered by changes
|
||||
to `plane/Dockerfile.base`, `pyproject.toml`, or `uv.lock`). It publishes
|
||||
`base-<short sha>` and `base-content-<input hash>` tags, plus their `-browser`
|
||||
variants, and prints immutable digest references. Content tags identify build
|
||||
inputs; they do not claim reproducible upstream apt/PyPI resolution. The lockfile
|
||||
triggers refreshes but pip installs the declared runtime constraints, not a uv
|
||||
frozen environment. Deliberate upstream refreshes use manual dispatch.
|
||||
- After both targets publish, ensure `ousast-task-base` is **public** in GHCR's
|
||||
package settings. Copy the workflow summary's JSON into `plane/task-base.digest`
|
||||
and submit that pin update for normal review. CI does not commit on your behalf.
|
||||
The initial null pins mean **unpublished**, not a usable bootstrap image.
|
||||
- A pin update or `plane/Dockerfile.*`/root Dockerfile change on main runs **Engine
|
||||
image**. For source, entrypoint or retention-installer changes, dispatch it once
|
||||
for the desired revision, or push a `v*` release tag. Source-only merges no longer
|
||||
publish images automatically. Never rebuild per task/run. All builds use scoped
|
||||
GHA caches; pin checks precede builds and require anonymous access plus identical
|
||||
base-layer ancestry for the browser variant.
|
||||
- Final package names must also be public in GHCR. Publication checks anonymous
|
||||
access after logout and fails if the package is private. Set visibility on first
|
||||
publication and rerun as needed. Deploy only the reported `image@sha256:...`
|
||||
references in the cluster profile/Task manifests, never `main` or a release tag.
|
||||
Publication does not automatically change deployed Task pins.
|
||||
|
||||
For a local task build, pass `TASK_BASE_TAG` and `TASK_BASE_DIGEST` from the base
|
||||
pin; search instead takes `TASK_BROWSER_TAG` and `TASK_BROWSER_DIGEST`. Replay's
|
||||
`ENGINE_REF` defaults to its local engine stage; set it to the published engine
|
||||
reference to reuse exactly those layers. When building `base-browser` locally,
|
||||
pass `TASK_BASE_REF=<full base tag@digest>` for the same ancestry as CI.
|
||||
|
||||
The operator's capacity rule is **all images in rotation, unpacked, together
|
||||
at most approximately 9 GiB**, including rollback revisions. Working builds take
|
||||
priority over minimizing an individual executor image; its previous 2.13 GB
|
||||
unpacked (~1.98 GiB) was acceptable. Retire unused Task templates and image
|
||||
revisions before rotating a large base/Joern generation. Shared layers occupy
|
||||
storage once where the runtime deduplicates them, but do not assume that turns
|
||||
a sum above 9 GiB into an acceptable rotation. Record both the sum of unpacked
|
||||
image sizes and actual unique-layer storage. Never prune images used by live tasks.
|
||||
|
||||
Expected footprint (planning estimates only; no images built for this change):
|
||||
|
||||
| Image | Expected unpacked size | Basis / uncertainty |
|
||||
| --- | --- | --- |
|
||||
| Shared base | about 1.3–1.9 GiB | ~454 MB inherited AX layers + ~281 MB JDK baseline + npm/Composer/Maven, language runtimes and venv/extras; apt dependency closure unmeasured |
|
||||
| Browser base | about 1.65–2.45 GiB | Shared base + roughly 350–550 MB Chromium/dependencies |
|
||||
| Search executor/verifier | about 1.7–2.5 GiB | Browser base + application; previous 2.13 GB unpacked is a baseline, not a ceiling |
|
||||
| Engine | roughly 3.5–6 GiB | Restored shared base + Joern; the supplied 2.16 GB compressed engine cannot establish its unpacked size |
|
||||
| Replay | engine + a few KiB | Same engine digest plus replay entrypoint |
|
||||
| VM/kind openultrasast | roughly 3.5–6 GiB plus benchmark assets | Same base/Joern, runtime extras shared, benchmark payload retained |
|
||||
|
||||
An engine + search rotation therefore sums to about **5.2–8.5 GiB** before
|
||||
rollback images. Adding replay as another unpacked image brings the sum to about
|
||||
**8.7–14.5 GiB**; this rotation is not demonstrated to fit 9 GiB. Engine/replay
|
||||
share almost all layers, but the operator must confirm the accounting and measured
|
||||
footprint before selecting a rotation. Extra revisions and the VM image add to
|
||||
the budget if retained there.
|
||||
|
||||
The previous search history included a 1.37 GB apt layer and 281 MB JDK; no
|
||||
per-package apt breakdown is available. Removing system Gradle and docs/caches
|
||||
may save space, while retained build tools and shared Python extras cost space.
|
||||
These ranges are not measured savings or a capacity guarantee. After publication,
|
||||
measure unpacked images and unique layers for the entire intended rotation, and
|
||||
verify build recipes, PHP/JavaScript retention, runner startup and Chromium before
|
||||
rollout. There is no separate <1.5 GiB executor gate.
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stage an installer checkout with lean kind manifests; never modify the source checkout.
|
||||
|
||||
Usage: python ops/ax/prepare-substrate.py SOURCE EMPTY_DESTINATION
|
||||
The destination lives only for an install/render. Source/build inputs are symlinked;
|
||||
manifests are copied because the installer uses fixed paths (including the OTEL ConfigMap).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def prepare(source: Path, destination: Path) -> None:
|
||||
source = source.resolve()
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
if any(destination.iterdir()):
|
||||
raise ValueError(f"staging directory must be empty: {destination}")
|
||||
for entry in source.iterdir():
|
||||
if entry.name != "manifests":
|
||||
(destination / entry.name).symlink_to(entry, target_is_directory=entry.is_dir())
|
||||
shutil.copytree(source / "manifests", destination / "manifests")
|
||||
install = destination / "manifests/ate-install"
|
||||
shutil.copytree(install / "kind", install / "kind-upstream")
|
||||
overlay = Path(__file__).resolve().parent / "substrate-kind-lean"
|
||||
shutil.copytree(overlay, install / "kind", dirs_exist_ok=True)
|
||||
# The installer applies this file directly before it renders the bundle.
|
||||
# Keep the strategic-merge directive only in the overlay's patch file.
|
||||
config = yaml.safe_load((overlay / "ate-otel-config.yaml").read_text())
|
||||
config["data"].pop("$patch")
|
||||
(install / "kind/disabled-otel-config.yaml").write_text(yaml.safe_dump(config))
|
||||
(install / "kind/ate-otel-config.yaml").rename(install / "kind/otel-config-patch.yaml")
|
||||
(install / "kind/disabled-otel-config.yaml").rename(install / "kind/ate-otel-config.yaml")
|
||||
kustomization = install / "kind/kustomization.yaml"
|
||||
kustomization.write_text(kustomization.read_text().replace("path: ate-otel-config.yaml", "path: otel-config-patch.yaml"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
prepare(Path(sys.argv[1]), Path(sys.argv[2]))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user