From e979fb0d95a7450a295dd87ac5c631b00682bfc4 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:12:37 +0200 Subject: [PATCH] chore: remove retired specs and dead code, state the 2026-10-13 focus fossil-mcp dead-code over src found 72 candidates; 69 are referenced by tests, __all__ or other modules, 3 are not and are removed (cli._first_source, pairs._scorable, provider.openrouter._extract_message_content). Eight spec directories that steering already called retired or superseded leave the tree (history keeps them, commits named in overview.md): harnessx-removal, harnessx-self-improving-rulesets, openrouter-sast-harness, learning-harness, constrained-detector, three-stage-scan, real-world-vfc-slice, unseen-repo-evaluation. overview.md gets a current-state section: the valid benchmark numbers, what they say, and the four active lines in order. Co-Authored-By: Claude Fable 5.1 --- .kiro/specs/constrained-detector/brief.md | 340 -------- .kiro/specs/constrained-detector/design.md | 294 ------- .../constrained-detector/requirements.md | 172 ---- .kiro/specs/constrained-detector/spec.json | 27 - .kiro/specs/constrained-detector/tasks.md | 122 --- .kiro/specs/harnessx-removal/brief.md | 30 - .kiro/specs/harnessx-removal/design.md | 507 ------------ .kiro/specs/harnessx-removal/requirements.md | 125 --- .kiro/specs/harnessx-removal/spec.json | 22 - .kiro/specs/harnessx-removal/tasks.md | 62 -- .../blueprint-alignment.md | 96 --- .../dependency-audit.md | 116 --- .../design.md | 614 -------------- .../requirements.md | 120 --- .../research.md | 191 ----- .../spec.json | 22 - .../harnessx-self-improving-rulesets/tasks.md | 277 ------- .kiro/specs/learning-harness/brief.md | 113 --- .kiro/specs/learning-harness/design.md | 651 --------------- .kiro/specs/learning-harness/requirements.md | 156 ---- .kiro/specs/learning-harness/research.md | 137 ---- .kiro/specs/learning-harness/spec.json | 31 - .kiro/specs/learning-harness/tasks.md | 269 ------- .../openrouter-sast-harness/assessment.md | 325 -------- .../openrouter-sast-harness/decisions.md | 29 - .kiro/specs/openrouter-sast-harness/design.md | 762 ------------------ .../openrouter-sast-harness/requirements.md | 242 ------ .kiro/specs/openrouter-sast-harness/spec.json | 7 - .kiro/specs/openrouter-sast-harness/tasks.md | 303 ------- .kiro/specs/real-world-vfc-slice/brief.md | 58 -- .kiro/specs/real-world-vfc-slice/design.md | 376 --------- .../real-world-vfc-slice/requirements.md | 93 --- .kiro/specs/real-world-vfc-slice/research.md | 106 --- .kiro/specs/real-world-vfc-slice/spec.json | 25 - .kiro/specs/real-world-vfc-slice/tasks.md | 79 -- .kiro/specs/three-stage-scan/brief.md | 77 -- .kiro/specs/three-stage-scan/design.md | 529 ------------ .kiro/specs/three-stage-scan/requirements.md | 134 --- .kiro/specs/three-stage-scan/research.md | 156 ---- .kiro/specs/three-stage-scan/spec.json | 22 - .kiro/specs/three-stage-scan/tasks.md | 203 ----- .kiro/specs/unseen-repo-evaluation/brief.md | 32 - .kiro/specs/unseen-repo-evaluation/design.md | 518 ------------ .../unseen-repo-evaluation/requirements.md | 107 --- .kiro/specs/unseen-repo-evaluation/spec.json | 22 - .kiro/specs/unseen-repo-evaluation/tasks.md | 402 --------- .kiro/steering/overview.md | 35 +- src/openultrasast/cli.py | 7 - src/openultrasast/pairs.py | 5 - src/openultrasast/provider/openrouter.py | 7 - 50 files changed, 32 insertions(+), 9123 deletions(-) delete mode 100644 .kiro/specs/constrained-detector/brief.md delete mode 100644 .kiro/specs/constrained-detector/design.md delete mode 100644 .kiro/specs/constrained-detector/requirements.md delete mode 100644 .kiro/specs/constrained-detector/spec.json delete mode 100644 .kiro/specs/constrained-detector/tasks.md delete mode 100644 .kiro/specs/harnessx-removal/brief.md delete mode 100644 .kiro/specs/harnessx-removal/design.md delete mode 100644 .kiro/specs/harnessx-removal/requirements.md delete mode 100644 .kiro/specs/harnessx-removal/spec.json delete mode 100644 .kiro/specs/harnessx-removal/tasks.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/blueprint-alignment.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/dependency-audit.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/design.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/requirements.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/research.md delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/spec.json delete mode 100644 .kiro/specs/harnessx-self-improving-rulesets/tasks.md delete mode 100644 .kiro/specs/learning-harness/brief.md delete mode 100644 .kiro/specs/learning-harness/design.md delete mode 100644 .kiro/specs/learning-harness/requirements.md delete mode 100644 .kiro/specs/learning-harness/research.md delete mode 100644 .kiro/specs/learning-harness/spec.json delete mode 100644 .kiro/specs/learning-harness/tasks.md delete mode 100644 .kiro/specs/openrouter-sast-harness/assessment.md delete mode 100644 .kiro/specs/openrouter-sast-harness/decisions.md delete mode 100644 .kiro/specs/openrouter-sast-harness/design.md delete mode 100644 .kiro/specs/openrouter-sast-harness/requirements.md delete mode 100644 .kiro/specs/openrouter-sast-harness/spec.json delete mode 100644 .kiro/specs/openrouter-sast-harness/tasks.md delete mode 100644 .kiro/specs/real-world-vfc-slice/brief.md delete mode 100644 .kiro/specs/real-world-vfc-slice/design.md delete mode 100644 .kiro/specs/real-world-vfc-slice/requirements.md delete mode 100644 .kiro/specs/real-world-vfc-slice/research.md delete mode 100644 .kiro/specs/real-world-vfc-slice/spec.json delete mode 100644 .kiro/specs/real-world-vfc-slice/tasks.md delete mode 100644 .kiro/specs/three-stage-scan/brief.md delete mode 100644 .kiro/specs/three-stage-scan/design.md delete mode 100644 .kiro/specs/three-stage-scan/requirements.md delete mode 100644 .kiro/specs/three-stage-scan/research.md delete mode 100644 .kiro/specs/three-stage-scan/spec.json delete mode 100644 .kiro/specs/three-stage-scan/tasks.md delete mode 100644 .kiro/specs/unseen-repo-evaluation/brief.md delete mode 100644 .kiro/specs/unseen-repo-evaluation/design.md delete mode 100644 .kiro/specs/unseen-repo-evaluation/requirements.md delete mode 100644 .kiro/specs/unseen-repo-evaluation/spec.json delete mode 100644 .kiro/specs/unseen-repo-evaluation/tasks.md diff --git a/.kiro/specs/constrained-detector/brief.md b/.kiro/specs/constrained-detector/brief.md deleted file mode 100644 index 79f4835..0000000 --- a/.kiro/specs/constrained-detector/brief.md +++ /dev/null @@ -1,340 +0,0 @@ -# Brief: the detector optimizer - -## Problem - -`learning-harness` built an honest measurement layer with a closed family taxonomy, a class-aware pair -scorer, per-family detector configurations, a split enforced by construction, verifier-gated reporting, -and a journal with attribution. What it did **not** build is a search. `run_learning_round` is a -hand-rolled hill-climber: one proposal, one minibatch, one holdout check, a threshold rule, accept or -revert. Every defect four review rounds found in it was a defect of that improvised loop rather than of -the measurement underneath it. - -The maintainer's judgement, and the reason for this spec: the remaining work is not a sequence of -patches to that loop. It is adopting an interface and an algorithm that already exist, are published, -and are better than what we would arrive at by fixing ours one finding at a time. - -## What the current loop gets wrong, measured - -Round zero on vibe-py, `deepseek-v4-flash`, K = 5, $0.69, 23/30 pair-correct: - -| family | scorable | recall | silence | Youden | noise floor | -|---|---|---|---|---|---| -| injection | 20 | 0.75 | 0.90 | +0.65 | **11** | -| access_control | 3 | 1.00 | 1.00 | +1.00 | 3 | -| config_secrets | 3 | 1.00 | 1.00 | +1.00 | 1 | -| deserialization | 2 | 1.00 | 1.00 | +1.00 | 0 | -| output_encoding | 1 | 1.00 | 1.00 | +1.00 | 1 | -| path | 1 | 1.00 | 1.00 | +1.00 | 1 | - -Four defects follow from the loop's shape, not from its bugs: - -1. The acceptance rule cannot discriminate. `decide` compares raw deltas: any regression on train or - holdout rejects, a tie rejects. The measured floor says 11 of 20 injection pairs disagree with - *themselves* across five runs, so a delta over a minibatch of 8 is mostly noise. The spec built - `sign_test` and `reliable_change` for exactly this and the rule calls neither. -2. The holdout is an input to the decision. `decide` reads `holdout_delta`. Req 5.4 keeps the - *proposer* blind to the holdout, and then the *selector* reads it every round. Repeated rounds - selecting on holdout performance is search-time overfitting of the test set, the same leak class - this project has already measured once, one level up. -3. Noise is fought with repetition instead of diversity. K = 5 spends five rollouts per pair per - side estimating variance. The published alternative spends that budget on more candidates and lets a - frontier keep whatever wins somewhere. -4. The proposer is told a score, not what went wrong. `FailureFacts` carries outcomes, run strings - and counts. The single largest lever in the published work is textual feedback from the *evaluation - trace*, and our evaluator produces exactly the right material (which finding landed on which line, - whether it fell inside the labeled span, whether the fixed side leaked, which family was claimed) - and then throws all of it away. - -## What others built - -**GEPA, Reflective Prompt Evolution (Agrawal et al., arXiv:2507.19457, ICLR 2026 oral).** Algorithm 1, -read from the paper rather than from a summary: - -- Split `D_train` into `D_feedback` and `D_pareto`. Initialise the candidate pool `P = [Φ]` and score - the seed on every instance of `D_pareto`, giving a score **matrix** `S[candidate][instance]`. -- Each iteration: select a candidate from the Pareto frontier; select a module round-robin; run it on a - minibatch drawn from `D_feedback`; collect scores, traces and **textual feedback** via a feedback - function `μ_f`; have a reflection LM rewrite that module's prompt; re-run the mutated system on the - *same* minibatch. -- Accept if and only if the minibatch average improved. Only then is the candidate scored on all of - `D_pareto` and added to the pool with its parent recorded. -- Return the candidate with the best average on `D_pareto`. - -Algorithm 2, the selection: for each instance take the best score any candidate achieved; keep the -candidates that achieve it; drop strictly dominated ones; sample a survivor with probability -proportional to the number of instances it leads on. - -Three things about that are directly relevant to us. The acceptance gate is **cheap and noisy on -purpose** (a single minibatch comparison) because the frontier, not the gate, is what protects -against a bad accept: anything that wins on one instance is retained. Selection is **stochastic and -diversity-preserving**, which is what stops the search collapsing onto one lineage. And the expensive -full evaluation is spent only on candidates that already passed the cheap gate. - -Reported: 93% on MATH with a DSPy `ChainOfThought` program against 67% unoptimised, +10% over MIPROv2 -and +20% over GRPO with **35× fewer rollouts**; on IFBench, optimal prompts after 678 rollouts against -GRPO's 24,000. Sample efficiency is the claim that matters for us, because our rollouts cost money. - -**DSPy's optimizer interface.** `MIPROv2` proposes instruction and demonstration candidates and searches -them with Bayesian optimisation (Optuna TPE), scoring on minibatches and running a full validation pass -on the incumbent every `minibatch_full_eval_steps`, an explicit two-fidelity design for a noisy metric. -`dspy.GEPA` takes `metric`, `reflection_lm`, a budget (`auto` / `max_full_evals` / `max_metric_calls`), -`reflection_minibatch_size`, `candidate_selection_strategy` (`pareto` | `current_best`) and -`track_stats`, and its metric returns `Prediction(score, feedback)` rather than a float. - -**SIMBA, Stochastic Introspective Mini-Batch Ascent (`dspy.SIMBA`).** Two of its ideas fit our corpus -better than GEPA's do. It runs `num_candidates` (6 by default) program variants per example, buckets the -trajectories, and ranks examples by **max-to-min score gap**, deliberately spending the budget on the -instances where variants disagree most. On our corpus that is not a nuisance, it is the training signal: -the 11 injection pairs that disagree with themselves are precisely the pairs where a rule change can flip -an outcome, and uniform minibatch sampling would spend most of its budget on the 9 that never move. It -also alternates two strategies rather than round-robining blindly, `append_a_demo` (a successful -trajectory becomes a demonstration) and `append_a_rule` (the model introspects on the bucket and writes a -rule), which is exactly the split our levers already have between `counterexamples`/`hard_negatives` and -`checklist`/`prompt`. Candidates are sampled by softmax over historical scores at temperature 0.2, a -cheaper third option beside GEPA's `pareto` and `current_best`. - -**The reflection meta-prompt** (GEPA Appendix C) is short and tells us what the feedback function must -produce. It shows the reflector the current instruction, then "inputs, outputs and feedback" for the -minibatch, and asks it to infer the task, extract "all niche and domain specific factual information ... -as a lot of it may not be available to the assistant in the future", and name any generalizable strategy. -That is a prompt and a specification for `μ_f`: feedback has to carry the domain facts a detector -cannot re-derive (which sink, which span, which side leaked) rather than a verdict. - -**GEPA's merge** is gated strictly: two candidates are crossed only when they share a common ancestor, -optimised *disjoint* sets of prompts, are both Pareto-optimal, and both beat the ancestor's aggregate. -Worth having once we have per-lever candidates, and worth not having before then. - -**The standalone `gepa` library** (MIT) exposes `gepa.optimize(seed_candidate, trainset, valset, -task_lm, reflection_lm, max_metric_calls)` over a `GEPAAdapter` protocol of three methods -(`evaluate`, `make_reflective_dataset`, `propose_new_texts`), with a seed candidate that is just a dict -of component name to text. That adapter shape is the interface we want whether or not we take the -dependency: it is the smallest surface that separates *what a detector is* from *how it is searched*. - -## What this spec proposes - -Restate the detector as an optimisable program and put a published optimiser behind it. - -- A detector is a candidate: a dict of named text components (`prompt`, `checklist`, - `hard_negatives`, `counterexamples`) plus its frozen non-text settings (tools, budgets). That is - already what `FamilyConfig` holds; it becomes the seed candidate. -- The metric returns a score and a reason. `score_pair_family` gains a feedback string built from - what the evaluation already knows. This is the highest-value change in the spec and it is independent - of which optimiser sits on top. -- `D_pareto` is carved out of the train split. The holdout leaves the acceptance rule entirely and - is scored once, for reporting, when the search is over. This is a requirements change to Req 9.5 and - the most important honesty consequence of the redesign. -- An adapter protocol (`evaluate`, `make_reflective_dataset`, `propose_new_texts`) with two backends: - our own implementation of Algorithms 1 and 2 by default, so the core install keeps - `dependencies = []`, and `gepa` behind an optional extra so its improvements arrive for free. -- The Pareto pool replaces the accept/revert tree. `Archive` already records per-pair winners; it - becomes the frontier, and the journal's parent records become the ancestry GEPA needs. - -## Where the noise actually is, and why the architecture may be wrong - -The maintainer's objection, and I think it is correct: the harness treats the detector as a black box -emitting a noisy scalar, and everything follows from that (K runs, majority vote, a measured floor, a -regression budget, a sign test). That is the stance you take when you cannot see inside the thing you are -measuring. We can see inside it. - -The variance is not in the judgement. `temperature: 0` is sent and thinking is disabled, and 11 of 20 -injection pairs still disagree with themselves. A traced run shows why: over four steps the model chose -different files to read and different grep patterns, and reached its conclusion from different context -each time. The freedom is in the **trajectory**, and we built statistics to average over it instead of -removing it. - -This is exactly what DSPy exists to argue about, and three of its constructs attack the source rather -than the symptom. - -**Constraints as predicates, enforced at generation time** (`dspy.Assert`/`Suggest`, arXiv:2312.13382). -A constraint is a boolean function of the output; on failure the past output *and the error* are injected -into a retry. We already have the predicates (is the reported line inside a parsed function, does the -named sink occur at that line, is the claimed family in the taxonomy, did the run cite a tool result), -and we apply every one of them **after the fact, at scoring time, to decide whether to count a finding**. -Applying them at generation time to decide whether to *accept* one is free, needs no dependency, and is -the single highest-value change available. The paper reports near-total intrinsic constraint satisfaction -and 5–15% downstream gains. - -**Typed decomposition instead of one free-running loop.** Our detector is a single ReAct loop that decides -which files to read, what to grep, when to stop and what to report; every one of those is a variance -source. DSPy's thesis is that each step gets a signature (a typed input/output contract), and a little -control flow composes them. Ours would be roughly: locate candidate sink sites; trace whether an untrusted -value reaches each; check for a guard between source and sink; judge. **Several of those steps need no -model at all**. The CST, the entry-point mapper, `flows` and `obligations` already answer them -deterministically, and we are currently paying a model to re-derive facts we hold. - -**Best-of-N against a checkable reward, instead of a majority vote** (`dspy.Refine`, `BestOfN`). Both -spend K samples. A majority vote assumes the modal answer is right and discards the disagreement; Refine -ranks the N attempts by an explicit reward and feeds a hint from the failures into the next attempt. We -have checkable rewards for free. The same budget, used as evidence rather than as averaging. - -**Why this matters more for our models, not less.** GEPA and SIMBA are demonstrated with a strong -reflection model over a weaker task model, and their gains come from evolving instructions. A cheap model -fails on an under-specified task and does well on a narrow typed one. So the cheaper the detector model, -the more of the return sits in decomposition and constraints and the less in prompt evolution, which is -the opposite of the order this spec originally proposed. - -### The experiment, run (2026-09-06, `benchmarks/measurements/2026-09-06-trajectory-divergence.json`) - -Twelve injection pairs, both sides, K = 5, temperature 0, thinking disabled, recording the tool-call -sequence of every run. $0.23. - -**The intended statistic could not be computed. Across 120 runs there was not one repeated trajectory**. There were -4.67 distinct trajectories per vulnerable side and 4.92 per fixed side, out of 5, and 78 distinct tool-*name* -sequences ignoring arguments entirely. The detector is not sampling one measurement five times. It is running -five different investigations and voting on the results. - -Asked as a gradient instead, the relationship is monotone and clean: - -| trajectory similarity (Jaccard over tool calls) | run pairs | disagreed on the outcome | -|---|---|---| -| high (≥ 0.75) | 2 | 0 (0.0%) | -| mid (0.4–0.75) | 24 | 1 (4.2%) | -| low (< 0.4) | 94 | 29 (30.9%) | - -Three more facts from the same run: - -- The variance is a search problem, and the search is `grep_repo`. 555 grep calls against 181 `read_file`, - 71 `entry_points`, 43 `find_refs`, 13 `read_definition`. The model spends its budget guessing patterns to - locate a sink, while the CST, the entry-point mapper, `flows` and `obligations` already know where the - sinks are. -- All of the instability is on the vulnerable side. Every one of the twelve pairs produced exactly one - distinct finding set on the fixed side. The detector is reliably silent on the fix and unreliably vocal on - the bug, so the noise is entirely in detection and none of it in false positives. -- Findings converge far more than trajectories do. 78 distinct trajectories, but one to three distinct - finding sets per pair. Many different investigations reach the same answer; six of twelve pairs are stable - despite never once repeating a trajectory. - -**What this does to the noise architecture.** The floor is not measuring the detector's uncertainty about a -question. It is measuring how often an unconstrained search happens to reach the sink. A majority vote over -five runs is therefore an unweighted ensemble of five arbitrary search -procedures rather than variance reduction, and a regression budget derived from it is a budget for the search's luck. K runs, the floor, -the budget and the sign test are all statistical treatment for a structural problem, and the 0% and 4.2% -rows of that table say what happens when the structure is fixed instead. - -## Reflection on the approved specification - -Read against that measurement, `learning-harness` divides cleanly into a part that stands and a part that -was answering the wrong question. - -**Stands, and is worth more now than before.** The closed taxonomy; class-aware pair scoring; the split -enforced by construction; the unscorable vocabulary and its denominators; verifier-gated reporting; the -corpus repairs; the publication machinery; the per-pair archive. None of it depends on the noise framing, -and all of it is what made this experiment possible to run and cost $0.23. A measurement layer that can -answer "is our own instability structural?" in twenty minutes is the thing that was missing before. - -**Was answering the wrong question.** - -- Req 3.5, K ≥ 3 runs per pair. The premise is variance reduction over a repeated measurement. There is - no repeated measurement: 120 runs, 120 trajectories. K runs is an *ensemble of distinct search - procedures*, and a majority vote over it is an unweighted ensemble, not an estimate. It should either be - renamed and published as an ensemble, which changes what every family number means, or dropped once the - search is constrained. -- Req 8.2, the noise floor as the regression budget. The floor measures how often an unconstrained - search happens to reach the sink. Deriving a regression budget from it means the worse the search wanders, - the more collateral damage a round is licensed to do. That is backwards, and it is why injection's budget - is 11 of 20. -- Req 9.5, the acceptance rule. Already flagged for reading the holdout every round. Now also: it - compares deltas between two ensemble votes, and calls the difference an improvement. -- The sign test and `reliable_change`. Added this session in good faith; they test whether an ensemble's - vote moved, which is not the question either. - -**What the data adds that neither spec anticipated.** - -- The instability is entirely one-sided. All twelve pairs produced exactly one finding set on the fixed - side. Specificity is solid and recall is the noisy half, so the leak ceiling and the false-positive - machinery are sound and it is the recall numbers that carry the caveat. -- `grep_repo` is the variance engine. It accounts for 555 calls against 181 `read_file` and 71 `entry_points`. The - single highest-leverage change in either spec is not a prompt, a lever or an optimiser: **stop asking the - model to locate the sink.** The CST, the entry-point mapper, `flows` and `obligations` already know. Give - the detector the candidate sites and ask it the question only a model can answer. Does untrusted input - reach this one, and is there a guard. -- The loop is partly self-correcting, so this should be a large win rather than a marginal one: 78 - distinct trajectories collapse to one to three finding sets per pair, and half the pairs are already - stable despite never repeating an investigation. - -**Limits of the experiment, stated so the conclusion is not over-read.** Twelve pairs, one family, one -slice, one model. The high-similarity bucket holds only two run pairs, so its 0% is decoration; the -defensible number is the mid bucket's 4.2% over 24 run pairs against 30.9% over 94. And the direction of -causation is inferred, not established. Similar trajectories may be a symptom of easy pairs rather than a -cause of agreement. The way to settle that is the constrained detector itself: fix the investigation, and -if the floor does not collapse, this reading was wrong. - -## The constraint that outranks the algorithm - -Both algorithms assume a dataset. GEPA splits `D_train` into `D_feedback` and `D_pareto`; SIMBA's default -batch size is 32. **Our injection family has 12 train teachers** (fewer than one SIMBA batch), and after a -GEPA split roughly six instances on each side. Every other family has one to three. - -So the honest statement of what this spec can and cannot buy: GEPA and SIMBA are sample-efficient in -*rollouts*, not in *instances*. They will not shortcut a corpus of twelve. The optimiser is worth building -because the current loop is the wrong shape and will stay wrong at any corpus size, but the binding -constraint on results is the train split, and it will remain so after this spec ships. Two things move it, -neither of which is an algorithm: reviewing agent-vfc rows above tier `title` (29 rows, currently unable to -teach anything), and harvesting more pairs per family. - -## Scope - -In: the search over a family's text components, the feedback function, the candidate pool and its -selection, the budget accounting, and the artifacts that make a run reproducible. - -Out, and deliberately: the taxonomy, the classifier, the pair scorer's *outcome* rules, the verifier -boundary, the write roots, redaction, the three gates. Those are the measurement, and a search whose -objective is not trustworthy is worse than no search. Nothing here relaxes them. In particular, the -proposer stays blind to the holdout, verifier code and gold labels stay outside every write root, and -no pattern or policy is written by a model. - -## Decisions for the maintainer - -0. Is the optimiser even the next thing to build? Proposed no, and this reverses the order above. - Constraints at generation time, typed decomposition and best-of-N attack the variance at its source; an - optimiser only tunes text around a program whose freedom is the problem. Proposed sequence: measure where - the variance is, constrain generation, decompose the loop, then optimise, and at a trainset of twelve the - optimiser that fits is `BootstrapFewShot` (collect traces that scored well, install them as demonstrations - in a lever we already have) rather than an evolutionary search with nothing to search over. - -0b. Adopt DSPy itself? Proposed no. Our detector is not a DSPy program. It is an agentic tool loop over - a repository with curated tools, redaction on every prompt, path clamping, a "no findings unless a tool - was called" rule and a final-answer turn, several of which are security properties rather than - conveniences. Handing that loop to `dspy.ReAct` and the calls to litellm would put them behind someone - else's abstraction, and DSPy could only ever be an optional extra under `dependencies = []`, which means - two code paths for the thing that *is* the product. What DSPy would buy (signatures, adapters, - multi-module composition), we barely need at one module per family. The optimisers are available without - the framework: `gepa` ships standalone under MIT, and SIMBA's two ideas are a hundred lines of logic. - Proposed instead: take the interface shape and the algorithms, not the framework. - -1. Reimplement Algorithms 1 and 2, take `gepa` as an extra, or both? Both is proposed: ours by - default (zero-dependency, and the security properties are ours to own), `gepa` behind an extra for - comparison. The cost is one adapter and one conformance test that both backends must pass. -2. Drop the holdout from the acceptance rule? Proposed yes, and it amends approved Req 9.5. The - holdout becomes what its name says. -3. Sample instances by disagreement (SIMBA) or uniformly (GEPA)? Proposed SIMBA's ranking. Our measured - floor makes it the better fit, and it turns the noise floor from an obstacle into an instance sampler. - -4. K = 1 inside the search, K ≥ 3 only for published numbers? Proposed yes. That is roughly a 5× - budget reclaim spent on more candidates, which is where the sample efficiency comes from. -5. Does the reflection model differ from the detector model? The published work uses a stronger - reflector than the task model. We have `deepseek-v4-pro` configured as the judge; proposed to reuse it. -6. Is `access_control` still the family we care most about? It has one train teacher. Until agent-vfc - rows are reviewed above tier `title`, the optimiser cannot work on it, whatever its algorithm. - -## Open questions - -- Does `gepa` run against an OpenAI-shaped base URL that is not OpenAI (our DeepSeek endpoint), and what - does it pull in? Both must be checked before the extra is offered, under the no-undeclared-dependency rule. -- Our "module" granularity: is a module a family, or a lever within a family? Round-robin over levers is - the closer analogue, but the levers are not independent. A checklist edit and a prompt edit interact. -- The feedback function's failure mode is a reflection model that writes plausible prose about a trace it - misread. What check makes a feedback string answerable rather than persuasive? -- GEPA's acceptance is a single noisy comparison. Our floor says a single comparison is close to a coin - flip on this corpus. Does the frontier really absorb that at n ≈ 20 instances, or is our corpus simply - too small for the algorithm's assumptions? - -Sources: [GEPA (arXiv:2507.19457)](https://arxiv.org/abs/2507.19457) · -[DSPy GEPA overview](https://dspy.ai/api/optimizers/GEPA/overview/) · -[DSPy MIPROv2](https://dspy.ai/api/optimizers/MIPROv2/) · -[gepa-ai/gepa](https://github.com/gepa-ai/gepa) · -[DSPy SIMBA](https://dspy.ai/api/optimizers/SIMBA/) · -[dspy/teleprompt/simba.py](https://github.com/stanfordnlp/dspy/blob/main/dspy/teleprompt/simba.py) · -[DSPy Assertions (arXiv:2312.13382)](https://arxiv.org/pdf/2312.13382) · -[dspy.Refine](https://dspy.ai/api/modules/Refine/) · -[DSPy (arXiv:2310.03714)](https://arxiv.org/pdf/2310.03714) diff --git a/.kiro/specs/constrained-detector/design.md b/.kiro/specs/constrained-detector/design.md deleted file mode 100644 index e94c96b..0000000 --- a/.kiro/specs/constrained-detector/design.md +++ /dev/null @@ -1,294 +0,0 @@ -# Design Document: constrained-detector - -## Overview - -The detector stops searching. A deterministic pass enumerates the candidate sites inside a labeled region from the -semantic IR; the model answers one bounded, typed question per candidate with no tools and no choice of what to look -at; the predicates that decide whether a finding counts are applied while it is being made, with one retry. The -unconstrained tool hunter stays in place as the generalist and as the baseline this is measured against. - -Two measurements shape every decision below and neither was a guess. **120 runs produced 120 distinct trajectories**, -so the variance is the loop's freedom rather than the model's judgement. And the candidate generator was measured -before anything was built: the **pattern ruleset caps recall at 12.1%**, below the 64.9% the unconstrained loop -already reaches, while the **semantic IR caps it at 96.6%** with a median of ten sites per labeled function. - -### Goals - -- Two runs of the same detector on the same input make an identical sequence of model calls. -- The limit on what the detector can find is a published number, not an assumption. -- Real-project slices decide whether a change is kept; hand-written teaching code cannot justify one alone. -- The K-run and noise-floor machinery is re-measured and deleted if a constrained detector does not need it. - -### Non-Goals - -- Prompt optimisation, reflective evolution, any optimiser. Deferred to a later spec, on this feature's evidence. -- Changing the taxonomy, the classifier, the scorer's outcome rules, the split, the verifier boundary or the gates. -- Adopting DSPy as a dependency. The interface shape is borrowed; the framework is not. - -## Boundary Commitments - -### This Spec Owns - -- `learning/candidates.py`, what a candidate is, how the IR enumerates one per family shape, and the ceiling report. -- `learning/judgments.py`, the typed answer, and the toolless call that produces it. -- `learning/predicates.py`, the checks that constrain a claim at generation time and score it afterwards, defined once. -- `learning/slices.py`, deciding versus development slices, the overfitting gap, and the cross-slice rejection rule. -- `run_constrained_detector` in `learning/detectors.py`, beside the existing `run_family_detector`. -- `ousast learning candidates`, and a detector selector on `score` and `baseline`. - -### Out of Boundary - -- `learning/families.py`, `classify.py`, `split.py`, `verifiers.py`, `canaries.py`, `journal.py`, `acceptance.py`, - `rounds.py`, `publish.py` keep their contracts. `scoring.py` is touched only to import predicates it already - implements inline, and a test asserts its behaviour is unchanged. -- `tool_hunter.run_tool_hunter` keeps its contract and its callers. It is not removed while it scores higher on any - deciding slice. -- The pair corpus, its labels, tiers and splits. The three gates. - -### Allowed Dependencies - -- Core: standard library only. `dependencies = []` is unchanged. -- Optional: the semantic extra for the IR. Without it, every family reports `unsupported_language` and scores nothing. -- The chat endpoint already resolved by `learning/endpoint.py`. No new provider, no new key. - -### Revalidation Triggers - -- The candidate ceiling on a deciding slice falling below the unconstrained loop's detection rate on that slice. -- The re-measured trajectory count per pair exceeding one. -- Any change to `scoring.py`'s outcome rules, which would invalidate the baseline this is compared against. - -## Architecture - -### Existing Architecture Analysis - -`run_family_detector` composes `run_tool_hunter` with a family's prompt, checklist, curated tools and context files. -The hunter loop is a ReAct loop: the model calls tools until it stops, then emits findings, and a final turn asks for -JSON when it ended in prose. `_overlay_scan` already parses both sides of a pair and returns function ranges; -`semantic/ir.py` already exposes `FunctionIR` with `calls: tuple[CallSite, ...]` and `binds: tuple[Bind, ...]`, -each carrying argument texts, the names they mention and whether they are constant. `hunter_tools.obligations` and -`flows` already answer the guard and reachability questions deterministically. Nothing new has to be parsed. - -### Architecture Pattern & Boundary Map - -```mermaid -flowchart TD - R[Region: path, function, admitted families] --> E[candidates.enumerate_candidates] - IR[(semantic IR: calls, binds)] --> E - OB[(obligations, flows, entry_points)] --> E - E -->|"Candidate[] sorted, bounded"| J[judgments.judge_candidates] - J -->|"one typed call per batch, no tools"| M[[chat endpoint]] - M --> P[predicates.check] - P -->|violation| J - P -->|clean| F[StaticFinding: family, detector, verifier tags] - F --> S[learning.scoring.score_pair_family] - P -.->|same definition| S - E --> C[CandidateReport: ceiling per family per slice] -``` - -The dotted edge is the point of Requirement 4: one definition of the predicates, used to constrain and to score. - -### Technology Stack - -| Concern | Choice | Why | -|---|---|---| -| Candidate enumeration | `semantic/ir.py` `FunctionIR.calls` and `.binds` | 96.6% ceiling measured; the ruleset's 12.1% is below the current baseline | -| Guards in scope | `hunter_tools.obligations`, `mapping.analyze_entry_points` | already deterministic, already used by the obligations checker | -| Typed answer | `json_object` on the DeepSeek adapter, one array per batch | the provider has no schema mode; the predicates are the schema | -| Retry on violation | one bounded retry with the violation injected | DSPy Assertions' backtracking, without the framework | -| Determinism | candidates sorted, batches fixed, no tools | Req 2.4; the model's text may vary, the call sequence may not | - -## File Structure Plan - -### Directory Structure - -``` -src/openultrasast/learning/ -├── candidates.py # NEW Candidate, CandidateKind, enumerate_candidates, ceiling, CandidateReport -├── judgments.py # NEW Judgment, judge_candidates, batching, the typed prompt -├── predicates.py # NEW PREDICATES, check_claim, violation reasons -├── slices.py # NEW DECIDING_SLICES, DEVELOPMENT_SLICES, overfitting_gap, cross_slice_verdict -├── detectors.py # MOD run_constrained_detector beside run_family_detector -└── scoring.py # MOD imports predicates; outcome rules unchanged, asserted by the recorded baseline -``` - -### Modified Files - -- `src/openultrasast/learning/detectors.py`, `run_constrained_detector(root, region, config, *, client, model, - taxonomy, ranges)`; `FamilyConfig` gains nothing, because a constrained detector is configured by the same - prompt/checklist/limits and reads no tools. -- `src/openultrasast/learning/scoring.py`, `_hits`, `_others` and `_fabricated` delegate to `predicates`; the - family metrics gain nothing. A test asserts the recorded baseline numbers are reproduced exactly. -- `src/openultrasast/learning/publish.py`, the overfitting gap and the per-slice table (Req 3.1, 3.4). -- `src/openultrasast/config.py`, `[learning] detector = "hunter" | "constrained"`, default `hunter` until the - comparison says otherwise. -- `src/openultrasast/cli.py`, `learning candidates`; `--detector` on `score` and `baseline`. -- `pyproject.toml`, unchanged. - -## System Flows - -### One region, constrained - -```mermaid -sequenceDiagram - participant D as run_constrained_detector - participant E as enumerate_candidates - participant J as judge_candidates - participant P as predicates - D->>E: root, region, family, ranges - E-->>D: Candidate[] (sorted by path, line, name; bounded) - alt no candidates - D-->>D: record no_candidate, return [] - end - loop each fixed-size batch - D->>J: batch, config, client - J-->>D: Judgment[] (typed) - D->>P: judgment, candidate, ranges, taxonomy - alt violation - D->>J: retry once, violation injected - J-->>D: Judgment[] - D->>P: re-check - end - P-->>D: clean claims or a recorded discard - end - D-->>D: StaticFinding[] tagged family, detector, verifier -``` - -## Requirements Traceability - -| Criteria | Delivered by | -|---|---| -| 1.1, 1.2 | `candidates.enumerate_candidates`, `CandidateKind` per family shape | -| 1.3 | `candidates.ceiling`, `ousast learning candidates`, `benchmarks/measurements/*-candidate-ceiling.json` | -| 1.4 | `enumerate_candidates` returns `Unsupported` when `parse_file(...).parse_ok` is false | -| 1.5 | `no_candidate` joins the unscorable vocabulary in `scoring.unscorable_reason` | -| 1.6 | `MAX_CANDIDATES_PER_CALL`, `CandidateReport.discarded` | -| 2.1, 2.2, 2.3 | `judgments.judge_candidates`, `Judgment`, `_typed_prompt`; findings built from `Judgment` only | -| 2.4 | `test_two_runs_make_an_identical_sequence_of_calls` over a recording client | -| 2.5 | `StaticFinding.tags` gains `candidate:::` | -| 3.1, 3.4 | `slices.overfitting_gap`, `publish._slice_table` | -| 3.2, 3.3 | `slices.DECIDING_SLICES`, `slices.cross_slice_verdict` | -| 3.5 | `learning candidates --slice` over every slice; the artifact carries per-slice ceilings | -| 3.6 | `slices.teachable(slice)` and a note emitted with every published number | -| 3.7 | `NON_GATING_FAMILIES` unchanged; `publish` reports memory separately, never inside a web aggregate | -| 4.1, 4.2, 4.3 | `predicates.PREDICATES`, `check_claim`, one retry in `run_constrained_detector`, `discarded` tally | -| 4.4 | `scoring._hits`/`_fabricated` call `predicates`; the baseline-reproduction test pins no drift | -| 5.1, 5.2 | the trajectory script and `run_baseline` re-run against the constrained detector | -| 5.3, 5.4 | the published floor names the `learning-harness` requirements it would amend; K is labelled | -| 6.1 | `score_pair_family` unchanged; the constrained detector only produces findings | -| 6.2, 6.3 | the comparison artifact, per slice, against 64.9% / 12.3% / 52.6% and $0.030 / $0.136 | -| 6.4, 6.5 | the comparison's headline rule; `[learning] detector` keeps `hunter` selectable | -| 7.1 | `resolve_chat_endpoint` unchanged; `learning_endpoint_unavailable` recorded | -| 7.2 | the client's `cost_usd` meter, already wired | -| 7.3 | the committed gate baseline test | -| 7.4 | `has_semantic_extra` guard; `unsupported_language` without it | -| 7.5 | `redact_secrets` on every prompt and persisted trajectory | - -## Components and Interfaces - -### candidates.py - -```python -CandidateKind = Literal["call", "bind", "operation", "config"] - -@dataclass(frozen=True) -class Candidate: - path: str - line: int - kind: CandidateKind - name: str # the callee, the bound name, or the operation - text: str # the site as written, clamped - arg_texts: tuple[str, ...] - arg_names: tuple[tuple[str, ...], ...] - binding_chain: tuple[str, ...] # what the IR resolved for the arguments, nearest first - guards: tuple[str, ...] # decorators and in-body guards in scope, from obligations - function: str - - @property - def id(self) -> str: ... # f"{path}:{line}:{name}", the tag a finding carries - -FAMILY_SHAPE: dict[str, tuple[CandidateKind, ...]] = { - "injection": ("call",), "path": ("call",), "deserialization": ("call",), - "untrusted_destination": ("call",), "prototype": ("call", "bind"), - "access_control": ("operation",), "config_secrets": ("bind", "config"), - "output_encoding": ("call",), "memory": ("call",), "unknown": ("call", "bind"), -} -MAX_CANDIDATES_PER_CALL = 12 # p90 of the measured distribution is 22; two batches covers all but the tail - -def enumerate_candidates(root, region, family, *, taxonomy, limit=MAX_CANDIDATES_PER_CALL) -> CandidateSet -def ceiling(cases, *, taxonomy) -> CandidateReport # offline, no model -``` - -`CandidateSet` carries `candidates`, `discarded`, and `reason` (`""`, `unsupported_language`, `no_candidate`). -Ordering is `(path, line, name)`, which makes the batches and therefore the call sequence deterministic. - -### judgments.py - -```python -@dataclass(frozen=True) -class Judgment: - candidate_id: str - verdict: bool - source: str # the expression carrying untrusted input, or "" - guard: str # the guard found, or "" meaning none - family: str - rationale: str - -def judge_candidates(candidates, config, *, client, model, taxonomy, violation=None) -> tuple[Judgment, ...] -``` - -One call per batch, `tools=[]`, `json_object=True`. The prompt is the family's prompt and checklist plus the batch -rendered as data; `violation` is the single retry's injected text. Answers that name a candidate id not in the batch -are dropped before the predicates run, because a fabricated id is not a claim about anything. - -### predicates.py - -```python -@dataclass(frozen=True) -class Violation: - predicate: str - detail: str - -def check_claim(judgment, candidate, *, ranges, taxonomy) -> Violation | None -def inside_labeled_function(finding, spans) -> bool # used by scoring._hits -def is_known_family(tag, taxonomy) -> bool # used by scoring._fabricated -``` - -Named predicates: `line_in_parsed_function`, `sink_at_line`, `family_in_taxonomy`, `source_in_function`. The last -two of the three helpers are the ones `scoring.py` already implements inline; moving them here is a refactor whose -only observable effect must be none, which the recorded baseline pins. - -### slices.py - -```python -DECIDING_SLICES = ("agent-vfc", "vfc-js", "vfc", "github") # harvested from real projects -DEVELOPMENT_SLICES = ("vibe-py",) # hand-written; cannot justify a change alone - -def overfitting_gap(per_slice: Mapping[str, FamilyMetrics]) -> float -def cross_slice_verdict(before, after) -> Verdict # rejects an improvement that regresses a deciding slice, by name -def teachable(slice_name: str, cases) -> bool # False while every row of a slice is below a gating tier -``` - -## Error Handling - -| Condition | Behaviour | Recorded as | -|---|---|---| -| IR cannot parse the side | no candidates, no model call | `unsupported_language` | -| Labeled function has no call or bind | no model call | `no_candidate` | -| Region exceeds the batch bound | judge the first `limit`, count the rest | `CandidateReport.discarded` | -| Predicate fails twice | claim discarded | `predicates.discarded[]`, published | -| Model names a candidate id not in the batch | answer dropped before predicates | `fabricated_candidate` tally | -| Endpoint unavailable | score nothing | `learning_endpoint_unavailable` | -| Cost cap crossed | stop, keep what was measured | `cost_cap_exceeded` | - -## Migration and Phases - -1. Candidates and the ceiling, offline. `candidates.py`, `learning candidates`, the per-slice ceiling artifact - including the real-world slices (Req 3.5). No model call, so the phase is free and its result can still cancel - the design. If the ceiling on a deciding slice is below that slice's current detection rate, stop here. -2. Predicates extracted. `predicates.py`, `scoring.py` delegating, the baseline-reproduction test. -3. Judgment and the constrained detector. `judgments.py`, `run_constrained_detector`, the determinism test. -4. Slices and reporting. `slices.py`, the overfitting gap, `publish` per slice. -5. The comparison. Constrained against unconstrained on every slice, the re-measured trajectory count and noise - floor, and the named `learning-harness` amendments if the floor collapses. - -Phase 1 is deliberately able to kill the design before any money is spent on it. diff --git a/.kiro/specs/constrained-detector/requirements.md b/.kiro/specs/constrained-detector/requirements.md deleted file mode 100644 index 60170e4..0000000 --- a/.kiro/specs/constrained-detector/requirements.md +++ /dev/null @@ -1,172 +0,0 @@ -# Requirements Document - -> **Status 2026-10-02:** superseded by `model-grounded-detection` (spec.json `superseded_by`). 4 of 20 tasks were done and the tasks were never approved; none will be. Kept as decision history; nothing in src, tests or CI points here. - -## Introduction - -The detector's instability is structural, not statistical. Measured on 2026-09-06 over twelve injection pairs at -temperature 0 with thinking disabled, **120 runs produced 120 distinct tool-call trajectories**, not one repeat, -and outcome disagreement tracks trajectory divergence (30.9% at low similarity against 4.2% at mid). The model -spends its budget searching: 555 `grep_repo` calls against 71 `entry_points`, guessing where sinks are while the -semantic IR already knows. So `learning-harness`'s K runs are an ensemble of distinct search procedures rather than -repeated measurements, and the noise floor it derives a regression budget from measures how often an unconstrained -search happens to reach the sink. - -This feature removes the freedom instead of averaging over it. A deterministic pass enumerates the candidate sites -inside the labeled region from the IR; the model is asked one bounded, typed question per candidate with no tools -and no choice of what to look at; and the predicates the scorer already applies after the fact are enforced at -generation time with a bounded retry. The candidate generator was measured before anything was built: the pattern -ruleset caps recall at 12.1%, below the 64.9% the unconstrained loop already reaches, while the semantic IR caps it -at 96.6% with a median of ten sites per labeled function. - -Baseline to beat, `deepseek-v4-flash`, K = 5, 57 scorable web pairs: **detection 64.9%, leak 12.3%, pair-correct -52.6%**, and 83.3% detection on hand-written vibe-py against 44.4% on agent-written agent-vfc, which is the gap -this feature must close rather than widen. Context and measurements: `brief.md`, -`benchmarks/measurements/2026-09-06-trajectory-divergence.json`, `2026-09-06-candidate-ceiling.json`. - -## Boundary Context - -- In scope: candidate enumeration from the semantic IR, per family shape; the typed, toolless judgment step; - constraint predicates enforced at generation time with a bounded retry; the evidence that decides whether the - K-run and noise-floor machinery still earns its place; per-slice reporting that keeps real-world code in front. -- Out of scope: the family taxonomy, the classifier, the pair scorer's outcome rules, the split, the verifier - boundary and write roots, redaction, the three gates, and the publication machinery, all of which stand and are - what made this feature's measurements possible. Also out: prompt optimisation, reflective evolution and any - optimiser (deferred to a later spec, on this feature's evidence); finding bugs in JavaScript interpreters, WebGL - renderers or browser engines; memory-safety fuzzing for C and C++; fine-tuning model weights; adopting DSPy as a - dependency. -- Adjacent expectations: `learning-harness` keeps owning the taxonomy, the scorer, the split and publication; - its Req 3.5 (K runs), 8.2 (noise floor) and 9.5 (acceptance) are amended by this feature only on evidence it - produces, never on argument. The pair corpus keeps owning labels, tiers and splits. The unconstrained tool hunter - stays as the generalist and as the baseline this feature is measured against. - -## Requirements - -### Requirement 1: Candidates come from the semantic IR, and the enumerator's ceiling is published - -**Objective:** As a maintainer, I want the sites a detector judges to be enumerated deterministically, so that the -model stops searching and the limit of what it can possibly find is a number rather than a hope. - -#### Acceptance Criteria - -1. The Constrained Detector shall enumerate candidate sites for a region from the semantic IR (call sites and - bindings), and shall not use the pattern ruleset as its candidate generator. -2. The Constrained Detector shall shape candidates by family: a call site for injection, path, deserialization, - untrusted destination and prototype; an operation on a protected resource with the guards in scope for access - control; a configuration assignment or literal for permissive configuration and secrets. -3. The Constrained Detector shall publish, per family and per slice, the fraction of labeled functions for which at - least one candidate was enumerated, and shall name that fraction as the ceiling on its recall. -4. If the semantic IR cannot parse a side, then the Constrained Detector shall record `unsupported_language` and - shall score nothing for that pair rather than falling back to a search. -5. If a labeled function yields no candidate site, then the Constrained Detector shall record `no_candidate` and - report that pair as unscorable with that reason rather than as a miss. -6. While enumerating, the Constrained Detector shall bound the candidates handed to one judgment call and shall - record the bound and the number discarded when a region exceeds it. - -### Requirement 2: One bounded, typed question per candidate, with no search - -**Objective:** As a maintainer, I want the model to answer only what a model is needed for, so that two runs of the -same detector on the same input take the same path. - -#### Acceptance Criteria - -1. The Constrained Detector shall call the model with the candidate site, its enclosing function, the binding chain - the IR resolved for its arguments, and the guards in scope, and shall offer it no tools. -2. The Constrained Detector shall require a typed answer per candidate: a verdict, the expression it believes - carries untrusted input, the guard it found or the statement that there is none, and the family. -3. The Constrained Detector shall derive every finding from a typed answer and shall never parse a finding out of - free prose. -4. When the detector is run twice on the same input with the same configuration, the sequence of model calls shall - be identical, and a test shall assert this. -5. The Constrained Detector shall record, for every claim, the candidate that produced it, so that a claim is - traceable to the site it was asked about. - -### Requirement 3: Real-world code decides, and the gap to teaching code is published - -**Objective:** As a maintainer, I want changes judged on code from real projects rather than on hand-written -teaching examples, so that the tool is not tuned for cases nobody has. - -#### Acceptance Criteria - -1. The Constrained Detector shall report every number per slice and shall never publish an aggregate across slices - without the per-slice numbers beside it. -2. The Constrained Detector shall designate the slices harvested from real projects (agent-vfc, vfc-js, vfc and - github) as the deciding slices, and the hand-written vibe-py slice as a development slice that cannot on its - own justify accepting a change. -3. If a change improves the development slice and regresses any deciding slice, then the Constrained Detector shall - reject it, and the rejection shall name the slice that regressed. -4. The Constrained Detector shall publish the **overfitting gap** (development-slice detection minus deciding-slice - detection) beside every headline number, and shall state that the baseline gap is 83.3% against 44.4%. -5. The Constrained Detector shall measure and publish the candidate ceiling separately for the real-world slices, - because the functions there are larger than the teaching corpus's and the enumerator's behaviour on them is not - implied by its behaviour on vibe-py. -6. While the deciding slices carry no pair a learning path may teach from, the Constrained Detector shall state that - with every number it publishes rather than presenting the development slice as if it were the corpus. -7. The Constrained Detector shall keep memory safety measured and non-gating, and shall not let the 176 vfc rows - dominate an aggregate that a reader would take as the tool's web-taxonomy performance. - -### Requirement 4: The predicates that score a finding also constrain it - -**Objective:** As a maintainer, I want the checks that decide whether a finding counts applied while it is being -made, so that a violation is corrected rather than silently discarded. - -#### Acceptance Criteria - -1. The Constrained Detector shall express as predicates the checks the scorer already applies: the reported line - falls inside a parsed function, the named sink occurs at that line, the claimed family is in the taxonomy, and - the claimed source expression appears in the enclosing function. -2. When a predicate fails, the Constrained Detector shall retry the judgment once with the failed answer and the - predicate that rejected it, and shall not retry more than once per candidate. -3. If a retry also fails a predicate, then the Constrained Detector shall discard the claim and record the predicate - that rejected it, and shall count that discard in a published tally. -4. The Constrained Detector shall apply the same predicates at generation time and at scoring time from one - definition, so the two cannot drift apart. - -### Requirement 5: The noise machinery is re-measured, and kept only if it earns its place - -**Objective:** As a maintainer, I want the K runs, the noise floor and the regression budget re-measured against a -constrained detector, so that machinery built for an unconstrained one is deleted rather than maintained. - -#### Acceptance Criteria - -1. The Constrained Detector shall re-run the trajectory measurement against itself and shall publish the resulting - distinct-trajectory count per pair. -2. The Constrained Detector shall re-measure the per-family noise floor at the same K as the baseline and shall - publish it beside the baseline's floor. -3. If the re-measured floor is zero for a family, then the Constrained Detector shall report that family's K-run - requirement and its regression budget as unnecessary and shall name the `learning-harness` requirements that - would be amended. -4. The Constrained Detector shall state, wherever a K-run number is published, whether the runs were repeated - measurements or an ensemble of distinct procedures. - -### Requirement 6: The constrained detector is measured against the unconstrained one, on the same corpus - -**Objective:** As a maintainer, I want one comparison that says whether this was worth building. - -#### Acceptance Criteria - -1. The Constrained Detector shall be scored by the existing class-aware pair scorer, with the same denominators, - the same unscorable vocabulary and the same split, and shall introduce no scoring rule of its own. -2. The Constrained Detector shall publish detection, leak and pair-correct rates against the recorded baseline of - 64.9%, 12.3% and 52.6%, per slice and in total. -3. The Constrained Detector shall publish its cost per correct pair against the baseline's $0.030 on vibe-py and - $0.136 on agent-vfc. -4. If the constrained detector's detection rate is below the unconstrained loop's on any deciding slice, then that - shall be published as the headline of the comparison rather than as a footnote. -5. The Constrained Detector shall keep the unconstrained tool hunter available and shall not remove it while it - scores higher on any deciding slice. - -### Requirement 7: Degradation, budget and boundaries - -**Objective:** As a maintainer, I want the same operational contract the rest of the tool has. - -#### Acceptance Criteria - -1. The Constrained Detector shall run behind the existing chat endpoint resolution and shall record - `learning_endpoint_unavailable` and score nothing rather than fall back to another provider. -2. The Constrained Detector shall meter its own spend through the client's meter and shall stop when a configured - cap is crossed, recording what it had spent. -3. The Constrained Detector shall leave the detection, map and local pair gate outputs byte-identical. -4. The Constrained Detector shall keep the core install at zero dependencies, and every optional capability it uses - shall degrade to a recorded reason without its extra. -5. The Constrained Detector shall pass every prompt and every persisted trajectory through redaction. diff --git a/.kiro/specs/constrained-detector/spec.json b/.kiro/specs/constrained-detector/spec.json deleted file mode 100644 index 3699cfd..0000000 --- a/.kiro/specs/constrained-detector/spec.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "feature_name": "constrained-detector", - "created_at": "2026-09-06T14:30:00Z", - "updated_at": "2026-09-08T00:00:00Z", - "language": "en", - "phase": "tasks", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": false - } - }, - "ready_for_implementation": false, - "depends_on": [ - "learning-harness" - ], - "approval_note": "2026-09-06 brief drafted at the maintainer's direction ('I'd rather spend time to design the exact DSPy style interface and evolutionary / statistical prompt optimization than fixing piece by piece'). Not approved; requirements not generated.; 2026-09-06 renamed from detector-optimizer to constrained-detector after the trajectory measurement: the brief's own conclusion is that the optimiser is not the next thing to build, and the spec should be named for what it delivers; 2026-09-06 requirements generated at the maintainer's direction, with the maintainer's emphasis recorded as Requirement 3 ('don't forget the emphasis on the real world bugs from the larger projects to avoid overfitting. we otherwise benchmark for cases that are not relevant'); 2026-09-06 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); 2026-09-06 design approved by the maintainer in session ('approve the design and generate the tasks'); 2026-09-08 SUPERSEDED by execution-grounded-detection: this session's measurements showed the noise architecture (K-runs, per-family floor, regression budget, sign test, evolve loop) is a statistical treatment for a structural problem, and that the field (Big Sleep, AIxCC finals, Clearwing) grounds detection in execution rather than in scoring against a labeled corpus. The taxonomy, the IR candidate enumerator (96.6% ceiling) and the obligation checker survive as the web/logic detection path to contribute to a Clearwing-based substrate.", - "superseded_by": "model-grounded-detection" -} diff --git a/.kiro/specs/constrained-detector/tasks.md b/.kiro/specs/constrained-detector/tasks.md deleted file mode 100644 index 3aa1b3c..0000000 --- a/.kiro/specs/constrained-detector/tasks.md +++ /dev/null @@ -1,122 +0,0 @@ -# Implementation Plan - -- [x] 1. The ceiling, offline: what a candidate detector could possibly find -- [x] 1.1 Candidate enumeration from the semantic IR, shaped by family - - `learning/candidates.py`: `Candidate` (path, line, kind, name, text, argument texts and names, the binding chain the IR resolved, the guards in scope, the enclosing function, and an `id` of `path:line:name`); `CandidateKind` of call, bind, operation, config; `FAMILY_SHAPE` mapping each family to the kinds that can carry its bug; `enumerate_candidates` returning a `CandidateSet` with the candidates, the count discarded past the bound, and a reason of empty, `unsupported_language` or `no_candidate`. - - Call and bind candidates come from `FunctionIR.calls` and `.binds`; operation candidates for access control come from the obligations facts with the decorators and in-body guards in scope; config candidates are bindings whose value is a literal in a configuration position. The pattern ruleset is never the generator. - - Ordering is `(path, line, name)` and the bound is a constant, so the batches a later task forms are determined by the input alone. - - Observable: on the shared obligations fixture every family shape yields the sites its docstring names; a file the IR cannot parse yields `unsupported_language` and no candidates; a labeled function with no calls or binds yields `no_candidate`; enumerating twice returns an identical sequence. - - _Requirements: 1.1, 1.2, 1.4, 1.5, 1.6_ - -- [x] 1.2 The ceiling report and the `learning candidates` command - - `candidates.ceiling(cases, taxonomy)` returns, per family and per slice, the number of labeled functions for which at least one candidate was enumerated, the sites-per-function distribution, and the rows that yielded nothing with their reason; `ousast learning candidates --catalog --slice --json` prints it and writes the artifact. No model is called at any point. - - Observable: the command runs offline with no endpoint configured; the artifact reproduces the recorded web-slice figures (ruleset 12.1%, IR 96.6%, median 10 sites) and adds the deciding slices; running it twice produces a byte-identical artifact. - - _Requirements: 1.3, 3.5_ - - _Depends: 1.1_ - -- [x] 1.3 The stop-or-go reading of the ceiling - - The artifact states, per deciding slice, the ceiling against that slice's recorded detection rate from round zero, and names the comparison as the condition on which this design continues. The reading is written into the spec's Implementation Notes with the numbers, whichever way it goes. - - Observable: the artifact carries agent-vfc, vfc-js, vfc and github ceilings beside 44.4% and the other recorded rates, and the notes state whether any deciding slice fails the condition. - - _Requirements: 1.3, 3.5, 3.7_ - - _Depends: 1.2_ - -- [ ] 2. One definition of the checks, used to constrain and to score -- [ ] 2.1 (P) Predicates extracted from the scorer without changing it - - `learning/predicates.py`: `Violation`, the named predicates `line_in_parsed_function`, `sink_at_line`, `family_in_taxonomy` and `source_in_function`, and `check_claim`; `scoring._hits`, `_others` and `_fabricated` delegate to them. - - Observable: the scorer's unit tests pass unchanged, and a test reproduces the recorded round-zero numbers for vibe-py and agent-vfc from the committed artifacts exactly, so the refactor's observable effect is none. - - _Requirements: 4.1, 4.4, 6.1_ - -- [ ] 2.2 The unscorable vocabulary gains `no_candidate` - - `no_candidate` joins the reasons `unscorable_reason` can return, distinct from `unresolved_label` (the corpus named a function nothing parses) and from `unsupported_language` (the tooling could not parse at all); it leaves every denominator and is listed by reason. - - Observable: a pair whose labeled function has no call or bind is reported unscorable with `no_candidate` and appears in the per-family unscorable tally, not as a miss. - - _Requirements: 1.5, 6.1_ - - _Depends: 1.1, 2.1_ - -- [ ] 3. The detector -- [ ] 3.1 Typed judgment, one bounded question per candidate, no tools - - `learning/judgments.py`: `Judgment` (candidate id, verdict, source expression, guard or empty, family, rationale) and `judge_candidates(candidates, config, client, model, taxonomy, violation=None)` making one call per fixed-size batch with `tools=[]` and JSON-object mode, the family's prompt and checklist as the system message, and the batch rendered as data. An answer naming a candidate id absent from the batch is dropped before any predicate runs. - - Observable: a scripted client shows one call per batch and no tool schema offered; a reply with a fabricated candidate id yields no judgment and increments the `fabricated_candidate` tally; a reply that is prose yields no judgment rather than a parsed guess. - - _Requirements: 2.1, 2.2, 2.3_ - - _Depends: 1.1_ - -- [ ] 3.2 `run_constrained_detector`, with the predicates enforced and one retry - - `detectors.run_constrained_detector` enumerates, batches, judges, checks each judgment with `check_claim`, retries a violating batch exactly once with the violation injected, discards what still violates, and returns `StaticFinding`s tagged `family:`, `detector:` and `candidate:`. `run_family_detector` is untouched and stays selectable. - - Observable: a judgment naming a line outside the parsed function is retried once and then discarded, and the discard is counted by predicate name; every finding carries the candidate id it came from; the unconstrained hunter still runs from the same configuration. - - _Requirements: 2.5, 4.1, 4.2, 4.3, 6.5_ - - _Depends: 2.1, 3.1_ - -- [ ] 3.3 Two runs make the same calls - - A recording client captures the full sequence of `(model, messages, tools)` for a run; two runs over the same region and configuration produce an identical sequence. - - Observable: the test fails if candidate ordering, batching or the retry condition becomes input-dependent in a way the input does not determine; it is run repeatedly in the suite to catch order instability. - - _Requirements: 2.4_ - - _Depends: 3.2_ - -- [ ] 3.4 Selecting the detector, and its degradations - - `[learning] detector = "hunter" | "constrained"` defaulting to `hunter`; `--detector` on `learning score` and `learning baseline`; without the semantic extra the constrained detector reports `unsupported_language` and scores nothing; without an endpoint it records `learning_endpoint_unavailable`; its spend goes through the client meter and stops at the cap. - - Observable: both detectors run from the same command over the same catalog; the extra-free matrix reports the recorded reason rather than failing; a run with no endpoint still writes its artifact. - - _Requirements: 7.1, 7.2, 7.4, 7.5_ - - _Depends: 3.2_ - -- [ ] 4. Real-world code decides -- [ ] 4.1 (P) Deciding and development slices, and the overfitting gap - - `learning/slices.py`: `DECIDING_SLICES` of agent-vfc, vfc-js, vfc and github; `DEVELOPMENT_SLICES` of vibe-py; `overfitting_gap` over per-slice metrics; `cross_slice_verdict(before, after)` rejecting by name any change that improves a development slice and regresses a deciding one; `teachable(slice, cases)` false while every row of a slice sits below a gating tier. - - Observable: a change that improves vibe-py and regresses agent-vfc is rejected with agent-vfc named; the gap over the recorded baseline reads 83.3% against 44.4%; `teachable("agent-vfc", …)` is false today and the reason is the tier. - - _Requirements: 3.2, 3.3, 3.4, 3.6_ - -- [ ] 4.2 Per-slice publication, and memory kept out of the web aggregate - - `publish` prints a per-slice table, the overfitting gap beside every headline number, the statement that the deciding slices teach nothing today, and the memory family separately from any figure a reader would take as web-taxonomy performance; no aggregate is printed without its per-slice rows. - - Observable: the published section carries a row per slice and refuses to print a cross-slice aggregate alone; the 176 vfc rows appear under their own heading; the regenerated roadmap section is byte-identical on a second run. - - _Requirements: 3.1, 3.4, 3.6, 3.7_ - - _Depends: 4.1_ - -- [ ] 5. The comparison, and what it retires -- [ ] 5.1 Constrained against unconstrained, per slice - - Both detectors scored by the same scorer over the same catalog at the same K, on every slice; detection, leak and pair-correct rates published against the recorded 64.9%, 12.3% and 52.6%, with cost per correct pair against $0.030 and $0.136. If the constrained detector is behind on any deciding slice, that is the headline of the artifact. - - Observable: one artifact holds both detectors' numbers per slice with the deltas, and its summary line names the deciding-slice result rather than the aggregate. - - _Requirements: 6.2, 6.3, 6.4_ - - _Depends: 3.4, 4.2_ - -- [ ] 5.2 The trajectory and the floor, re-measured - - The trajectory experiment re-run against the constrained detector, publishing distinct trajectories per pair; the per-family noise floor re-measured at the same K as the baseline and published beside it; every published K-run number labelled as repeated measurements or as an ensemble of distinct procedures. - - Observable: the artifact reports the distinct-trajectory count per pair for both detectors on the same twelve pairs, and the floors side by side. - - _Requirements: 5.1, 5.2, 5.4_ - - _Depends: 5.1_ - -- [ ] 5.3 What the floor retires, named - - If a family's re-measured floor is zero, the artifact states that its K-run requirement and its regression budget are unnecessary and names the `learning-harness` requirements that would be amended — 3.5, 8.2 and 9.5 — without amending them here. - - Observable: the artifact carries the per-family verdict and the named requirements, and the spec's Implementation Notes record the escalation to the maintainer. - - _Requirements: 5.3_ - - _Depends: 5.2_ - -- [ ] 5.4 Gates, matrices and the boundary - - Detection, map and local pair gate outputs byte-identical to the committed baseline; both test matrices green; the core install still resolving with no dependencies; every prompt and persisted trajectory redacted. - - Observable: the committed gate-baseline test passes in both matrices, and a planted key in a judgment prompt does not survive into the artifact. - - _Requirements: 7.3, 7.4, 7.5_ - - _Depends: 5.1_ - -## Task-plan review notes - -- Phase 1 is deliberately able to end the feature. Tasks 1.1 to 1.3 call no model, so the condition in 1.3 — a deciding slice whose ceiling is below its recorded detection rate — is reachable before any budget is spent. The ruleset variant of this design would have failed that condition at 12.1% against 64.9%. -- Task 2.1 is the only task that touches `scoring.py`, and its observable is that nothing changes. The recorded round-zero artifacts are the pin; if they cannot be reproduced exactly, the refactor is wrong and not the numbers. -- `(P)` marks the two tasks with no dependency on a model call: 2.1 and 4.1 can be built and reviewed while phase 1 runs. -- Task 3.3 is the one that would have caught the defect this whole spec exists for. It belongs in the suite, run repeatedly, not in a one-off measurement. - -## Implementation Notes - -- Task 1.1 (2026-09-06, RED first: 13 failing tests in `tests/test_candidate_enumeration.py`; then green). `learning/candidates.py`. Two things the fixtures taught: the IR emits a chained call twice at one line, once bare and once carrying the keyword arguments, so deduplication keeps the richest — a bare `filter_by` and `filter_by(user_id=owner)` are the same site and only one of them says what the constraint was. And `leaky` and `constrained` in the shared obligations fixture **both** discharge through `filter_by`; what separates them is that one binding chain ends at `request.args.get('owner')` and the other at `request.user.id`. The test asserts that contrast, which is the judgement the model is being narrowed to. -- Task 1.2 (2026-09-06, RED first: 3 failing tests; then green). `candidates.ceiling(cases, taxonomy, generator="ir"|"ruleset")` and `ousast learning candidates`. The `ruleset` arm exists so the comparison that rejected it stays a measurement rather than a remembered number. The command calls no model and its artifact is byte-identical on a second run. -- Task 1.3 (2026-09-06). **Verdict: go.** Measured offline across every slice, `benchmarks/measurements/2026-09-06-candidate-ceiling-all-slices.json`: - - | slice | ceiling | round-zero detection | headroom | - |---|---|---|---| - | agent-vfc (deciding) | **89.7%** | 44.4% | +45.3 | - | vfc (deciding, non-gating) | 84.7% | pending | — | - | vfc-js (deciding) | 52.9% | none recorded | see below | - | github (deciding) | 100.0% | none recorded | — | - | vibe-py (development) | 100.0% | 83.3% | +16.7 | - - The margin is largest on agent-vfc, which is the slice that matters most and the one the tool is actually for. Per family: config_secrets, deserialization, output_encoding and untrusted_destination 100%; access_control 90%; path 87.5%; memory 86.4%; injection 84.6%; prototype 71.4%; `unknown` 0% of two rows, which is what abstention looks like. - - - **A shape correction the measurement forced, before the detector existed.** `config_secrets` first read 63.6% with a bind-and-literal shape. A permissive default is as often an argument to a call — `cors({origin: '*'})`, `app.use(session({secure: false}))` — as it is a binding; adding call sites to that family's shape took it to 100% and recovered `new-erp-final-app-6e9644`, `zoonk-origin-bfbf67`, `melodix-playerprovider-089931` and `registry-proxy-worker-426847`. - - **Carry-forward, and it is not this feature's to fix.** vfc-js sits at 52.9% because eight of seventeen rows label a function the JavaScript CST names ``: `module.exports.publish = function () {}` and its relatives. The gap is in function *naming*, not in candidate generation, and it caps the existing hunter on that slice exactly as much. Same class as the `_declarator_start` defects found during learning-harness task 4.1. - - vfc's 21 `unsupported_language` rows are C and C++ files the IR declines; they are the non-gating memory slice and are already excluded from every web number. diff --git a/.kiro/specs/harnessx-removal/brief.md b/.kiro/specs/harnessx-removal/brief.md deleted file mode 100644 index 7c26932..0000000 --- a/.kiro/specs/harnessx-removal/brief.md +++ /dev/null @@ -1,30 +0,0 @@ -# Brief: remove HarnessX after the plane proves value - -Status: brief only, gated. Not started until `ai-service-plane` Requirement 6 records a met gate. - -## Inventory (read-only, 2026-09-29) - -- Code: `src/openultrasast/harness_ext.py` (97 lines) and `hunter_harness.py` (197) exist only for HarnessX; - guarded branches in `cli.py` (372-512), `fusion.py` (LLM panels, ~100 lines), `verify_judge.py` (judge path, - ~90), `stage_processors.py` (`host_under_harnessx`, ~40), `config.py` (`HarnessxConfig`). About 575 of - 30,243 lines (1.9%). Every path falls back deterministically when the extra is absent and records a - degradation (`harnessx_extra_unavailable`; fallbacks `run_hunter_pool`, `structural_verifier`, - `deterministic_panels`). `evolve`, `regress`, `benchmark`, `gate` and quick mode do not depend on it. -- Comment-only mentions (re-verified 2026-09-30): `gate.py:174`, `improve/evolve.py:9`, `slot_contract.py:7-13`. -- Packaging: `pyproject.toml` extra `harnessx` pinned to a git commit; mypy override; `uv.lock` entry. -- Tests: 14 files touch it; dedicated `test_harness_ext.py`, `test_hunter_harness.py`, `test_cli_hx_dispatch.py`, - `test_verify_judge.py`; `tests/conftest.py` fixture `assert_cold_of_harnessx`. -- Docs: README sections "Configuring the HarnessX agentic plane" and "The HarnessX self-improving cycle"; - `docs/examples.md` section 6; `docs/threat-model.md` egress and spend caps. -- Specs: `harnessx-self-improving-rulesets` (approved, 43/43 done, Requirement 1 is direct adoption); - `learning-harness` (30/33); `three-stage-scan` framed as its follow-up; `openrouter-sast-harness` (no - spec.json); mentions in four more; steering `overview.md:175`. - -## Shape of the removal - -1. Replace the three guarded paths with plane tasks (hunter pool -> `verify`; judge -> `judge`; panels -> - `judge`/`agree`) so their fallbacks become the only path, then delete the two modules, the config block, - the extra, the override and the fixture. -2. Retire the four specs' HarnessX requirements with a dated note, not a rewrite; update README and docs. -3. Gate: the plane increment met (recall and cost), and the replaced features measured no worse on the - development pins before deletion. diff --git a/.kiro/specs/harnessx-removal/design.md b/.kiro/specs/harnessx-removal/design.md deleted file mode 100644 index e60ca22..0000000 --- a/.kiro/specs/harnessx-removal/design.md +++ /dev/null @@ -1,507 +0,0 @@ -# Design Document - -## Overview - -HarnessX leaves the product in one reviewable sequence, and the plane's memory layer takes over what HarnessX was -meant to drive. Three facts from the code shape the design: - -- **No HarnessX capability is measured, and one of them never existed.** The LLM paths are reached only through - config keys, not CLI flags (`scan` has `path`, `--mode`, `--config`, `--fail-on`; `cli.py:163-167`), and - `benchmarks/` holds no run of any of them (no `hx-hunter:` finding, no `llm-panels` decision). The - `MetaAgent.evolve` proposer is only a docstring (`improve/evolve.py:8-9`, `harness_ext.py:7-8`). No code calls it. -- **The plane already does the LLM work better, and outside `ousast scan`.** `verify` (two passes, known callers - from `repo-facts`) plus `agree` (2-of-3 with the tie-break) met the ai-service-plane gate - (`benchmarks/independent/plane-increment-2.json`: 16/20 declared sites agreed, $0.0205 per candidate). The - standard-mode LLM hunter that users actually reach, the tool hunter at the MAP stage (`cli.py:631-640`, - `tool_hunter.resolve_hunter_client`), is not HarnessX and stays. -- **Everything deterministic already runs without the extra.** Every guarded branch falls back to a deterministic - function (`run_hunter_pool`, `verify_findings`, `fuse_finding`). Removal therefore makes the fallback the only - path. Quick mode, `benchmark`, `pairs`, `improve` and the gates never enter a guarded branch. - -So every HarnessX capability in `ousast scan` is **retired** with its deterministic path kept. The plane's `verify` -and `agree` tasks are the documented agentic path (`ousast plane run `). The proposer is **carried by the -memory layer**: a persistent store, keyed by repository and pin, that plane Runs write through a model-free -`remember` task. From that store a deterministic generator emits `RuleStatusEdit` proposals into the unchanged -`EvolveValidator` and gate. The improvement loop becomes a plane Run whose only Model-bound tasks are the verify -passes. - -## Architecture - -``` -Before After -ousast scan --mode standard ousast scan --mode standard - hunter_pool: HxScanOrchestrator | run_hunter_pool hunter_pool: run_hunter_pool (only path) - verify: verify_judge (llm-judge) | verify_findings verify: verification.verify_findings - fusion: llm panels | deterministic panels fusion: deterministic panels (fuse_findings) - MAP: tool hunter ([models] hunter, DeepSeek) MAP: tool hunter (unchanged) -harness_ext.py, hunter_harness.py, verify_judge.py deleted -stage_processors.host_under_harnessx deleted; deterministic pipeline kept -[harnessx] config, extra, mypy override, lock one load warning; extra/override/lock gone -improve: benchmark per-rule proposer (+ MetaAgent improve: benchmark proposer + memory proposer (--memory) - "plugs in" -- never implemented) - plane Run (per case) - facts -> va, vb -> agree -> vc -> final (existing) - alerts (new, no model) ------------------+ - remember (new, no model) <- facts, final, alerts, va/vb/vc - plane Run (loop tail, once) - measure -> propose -> improve(validate+gate) (no model) -~/ousast-results/plane/memory/ the store: facts by content, rows by repo + pin -``` - -New code: `src/openultrasast/plane/memory.py` (store, ingest, seed; host side), -`src/openultrasast/plane/tasks/{alerts,remember,measure,propose,improve}.py`, -`src/openultrasast/improve/memory.py` (deterministic proposal rules). None of it is in the reconciler. The -reconciler gains only `mark_done` (below). It is 465 lines today, and `tests/test_plane_reconciler.py:759-761` -keeps it under 500. - -## Components and Interfaces - -### 1. Fate of each HarnessX capability (Requirement 1) - -| Capability | Where today | Fate | What the user loses | -| --- | --- | --- | --- | -| LLM hunter pool (`HxScanOrchestrator`) | `hunter_harness.py:49-185`; gated in `cli.py:400-456` (`hx_hunter` at :405, orchestrator at :438-448, degradation at :452-455) | **Retired.** `run_hunter_pool` becomes the only standard-mode hunter pool. LLM hunting of a repository stays available in two places: the MAP-stage tool hunter in `ousast scan --mode standard/deep` with `[models] hunter` (`cli.py:631-640`, not HarnessX), and plane `verify` via `ousast plane run plane/runs/validation-46.yaml` | `hx-hunter::` findings from an anthropic/openai/litellm agent (`hunter_harness.py:155`). No run of this path is recorded under `benchmarks/`, so no measured TP is lost. | -| LLM judge | `verify_judge.py:22-63`; gated in `cli.py:406, 499-511` | **Retired.** `verification.verify_findings` is called directly and `verify_judge.py` is deleted (it is only the dispatcher and the judge). The plane's LLM verification is `verify` + `agree` (+ `vc`/`final` tie-break) | Per-finding `plausible/unsupported` verdicts that set `ACCEPTED/REJECTED` (`verify_judge.py:83-88`). Unmeasured. `[models] judge` (model-grounded-detection, `model/endpoint.py:195`) is a different capability and stays. | -| LLM fusion panels | `fusion.py:194-326` (`_fuse_with_llm_panels`, `_run_llm_panels`); gated in `cli.py:519-541` | **Retired.** The deterministic panels (`_panel_a`, `_panel_b`, `_reconcile`, `fusion.py:101-167`) become the only path. `fuse_findings_dispatch` is renamed `fuse_findings(findings, verifications, *, high_assurance, mitigated_ids)`. The plane counterpart of independent cross-examination is `agree` across independent passes with the 2-of-3 tie-break | Steel-man LLM panels (`decision_source: "llm-panels"`). Unmeasured. `fusion.json` for deterministic decisions is unchanged. | -| Deterministic stages hosted under HarnessX | `stage_processors.py:129-156` (`host_under_harnessx`); only caller is the skipped test `test_slot_contract.py:119-133` | **Retired, no replacement.** ax runs whole tasks, so a model-disabled in-loop host has nothing to host. `stage_processors`/`slot_contract` stay as the standalone deterministic pipeline that `test_gate.py:147-148` pins | Nothing. The function has no production caller. | -| LLM evolution proposer (`MetaAgent.evolve`) | docstring only: `improve/evolve.py:8-9`, `harness_ext.py:7-8` | **Carried by the memory layer** (Requirement 6): `improve/memory.py` rules over the store, fed into `run_round` as extra proposals | Nothing implemented is lost. | - -No CLI flag is HarnessX-specific, so Requirement 1.3 applies to config keys. A key that asked for a retired -capability makes `load_config` raise `RetiredConfigError(ValueError)`, and the CLI prints the message and exits 2. -A silent downgrade is not allowed: - -| Key | Message (abridged) | -| --- | --- | -| `[models] verifier` | "`[models] verifier` was retired with HarnessX on : LLM verification runs on the plane (`verify` + `agree`, `ousast plane run`, see ops/ax/README.md). Remove the key." | -| `[fusion] panel_model`, `[fusion] decider_model` | "... LLM fusion panels were retired; fusion is deterministic; independent LLM agreement is the plane's `agree` task ..." | - -`[harnessx]` is different (Requirement 3.3). It only tuned budgets and never requested a capability, so it loads -with **one** `logging.warning` on logger `openultrasast.config` that names the plane ("`[harnessx]` is ignored: -HarnessX was removed; agentic work runs on the ax plane with per-task budgets, see ops/ax/README.md"). The section -is otherwise ignored. `[models] hunter` keeps its meaning (tool hunter, obligations intent adjudication at -`cli.py:1237`), so it gets no message. With the extra installed it no longer starts a second, HarnessX hunter. The -release notes say so. - -Requirement 1.2 (moved capabilities have a CLI path, a Model per task and a fixture test) is met by the existing -`verify`/`agree` tasks (`ousast plane run`; Model `deepseek-flash` bound per Task; `tests/test_plane_verify.py`, -`tests/test_plane_scoring_parity.py`, `tests/test_plane_tiebreak.py`), and by the new memory and loop tasks below, each with its own fixture test. No -capability is moved into a new Model-bound task. - -### 2. Removal mechanics (Requirement 2) - -**Source.** - -- Delete `harness_ext.py` (97 lines), `hunter_harness.py` (197) and `verify_judge.py` (130). -- `cli.py`: drop the imports at :44, :46 and :94. Drop `harnessx_present`, `hx_hunter` and `hx_verify` (:400-406), - the orchestrator branch and its degradation (:438-456), and the verify degradation (:499-502). Call - `verify_findings(findings)` at :503-511. Drop `hx_fusion` and its degradation (:521-527). Call - `fuse_findings(findings, verifications, high_assurance=config.fusion.high_assurance)`. The `verifier_model` - local goes away. -- `fusion.py`: drop the import at :24, the `panel_model`/`decider_model`/`provider`/`use_harnessx` parameters, and - `_with_degradation`, `_fuse_with_llm_panels`, `_run_llm_panels` and `_text` (:229-338). Rewrite the module - docstring (:10-14) and update `__all__`. -- `stage_processors.py`: drop `host_under_harnessx` (:129-156) and its `__all__` entry, and rewrite the module - docstring (:10-11). `slot_contract.py:7-14` becomes "the zero-dependency substrate for the deterministic - pipeline". -- `config.py`: drop `HarnessxConfig` (:99-107), `ResolvedConfig.harnessx` (:201), `_load_harnessx` (:345-351) and - its call (:230). Drop `ModelConfig.verifier` (:37), `FusionConfig.panel_model/decider_model` (:123-124) and - their loaders (:266, :366-367). Add `RETIRED_KEYS` and the one-warning rule above. `harness.py:183` keeps the - `"verifier": None` literal in `model_roles`, so `harness.json` keeps its schema byte for byte (the key has always - been `null` in the baseline config). -- Comment-only mentions: `gate.py:174` ("a model-backed stage"), and `improve/evolve.py:8-9`, which now names the - memory proposer. -- `plane/Dockerfile.runner:23` loses "no HarnessX". - -**Packaging.** - -- Delete the `harnessx` extra and its comment (`pyproject.toml:14-22`) and the `harnessx.*` mypy override and its - comment (:67-71). -- `uv lock` regenerates `uv.lock`. The `harnessx` package (:1062), the extra (:2145-2146), the requirement - (:2169) and `provides-extras` (:2178) go. -- Extras that remain: `semantic`. There is no `all` extra. `pip install .` and `pip install '.[semantic]'` in a - fresh venv are the Requirement 2.2 check. -- No source or test imports a HarnessX transitive dependency. The only third-party imports are `yaml`, - `tree_sitter` and `pytest` (checked with an import scan), so dropping the lock entry cannot break an import. - -**Tests** (14 files plus `conftest.py`): - -| File | Action | -| --- | --- | -| `test_harness_ext.py`, `test_hunter_harness.py`, `test_verify_judge.py`, `test_cli_hx_dispatch.py` | **Delete.** Each exercises only HarnessX or its dispatcher. | -| `conftest.py` | Rename fixture `assert_cold_of_harnessx` to `assert_cold_import(code)`: a fresh interpreter runs `code`, with no module-name guard. `test_semantic_extra.py:28-35` is its remaining user (the tree-sitter check brings its own assertion). | -| `test_gate.py` | Delete `test_harnessx_stage_stays_within_tolerance_of_baseline` (:124-150) and the `has_harnessx` import (:23). Replace the two "never import" tests (:155-175) with `test_governance_planes_import_only_stdlib_and_declared_dependencies`, an allow-list (stdlib, `openultrasast`, `yaml`) over module-level imports of `GOVERNANCE_SCORING_BENCHMARK_PLANES`. It covers any extra without naming one. Update the docstring at :5-6. | -| `test_scan_pipeline_integrity.py` | Rename `test_standard_scan_extra_absent_matches_deterministic_baseline` to `test_standard_scan_with_a_hunter_model_keeps_deterministic_findings` (config `hunter` only). Rename `test_quick_scan_is_unaffected_by_the_extra` to `test_quick_scan_records_no_degradation`. Drop the `has_harnessx` monkeypatch and the `harnessx` parameter. Recompute the expected degradation set from the post-removal run, since `hunter_pool`/`verify` entries no longer exist. | -| `test_slot_contract.py` | Delete `test_host_under_harnessx_disables_the_model` and the imports at :16 and :22. Keep the rest. | -| `test_config.py` | Replace `test_harnessx_defaults_when_section_absent` and `test_load_config_reads_harnessx_section` with `test_retired_agentic_section_loads_with_one_warning` (caplog: exactly one record naming the plane) and `test_retired_llm_keys_fail_naming_the_replacement` (verifier, panel_model, decider_model). | -| `test_fusion.py` | Delete `test_dispatch_records_degradation_when_llm_requested_but_extra_absent` (:142-146) and the import at :16. Rename the dispatch calls to `fuse_findings`. | -| `test_tool_hunter.py` (:229, :266), `test_scan_deep.py` (:144) | Drop the `has_harnessx` monkeypatch line. | -| `test_semantic_extra.py` | Use `assert_cold_import`. | -| `test_module_audit.py` | The subsystem tuple at :31 becomes `("fusion", "mcp", "skills")`. A new assertion checks that the three deleted modules are absent from the audit. | - -**Module audit.** `model/audit.py:76-77` (`harness_ext`, `hunter_harness`) are deleted, and `:78` -(`stage_processors`) is reworded to "deterministic slot-contracted scan stages; pinned by the zero-dependency -guard in test_gate". Regenerate `benchmarks/measurements/2026-09-08-module-audit.json` in place with `to_dict(audit(SRC))`. -`test_the_manifest_matches_the_tree` pins that path. The diff shows three rows removed and no orphan. - -**Search (Requirement 2.4).** `grep -rni harnessx src tests pyproject.toml README.md` may match only lines tagged -`retired 2026-..`: - -- the `RETIRED_KEYS` entry and warning text in `config.py` -- the two retirement tests in `test_config.py` -- the one README line in the plane section - -Requirement 3.3 cannot be implemented or tested without naming the section, so these lines count as retirement -notes. The check is a test (`test_no_harnessx_outside_retirement_notes`) that fails on any untagged match. It -builds its pattern at runtime, so it does not match itself. - -### 3. Equality proof (Requirement 3) - -The spec's "`ousast regress`" is not a subcommand (`cli.py:133-251`). Regress exists only as the deep-mode -sandbox stage. On this project the regression command over the development pins is `ousast pairs` -(README.md:38): the vuln-vs-fixed replay over local fixtures and vendored pinned VFC pairs, offline by default. -The benchmark replay is README.md:35's manifest. The baseline also covers the two paths the removal touches -directly: one standard-mode scan (verify and fusion) and one `improve --dry-run` (evolve). Everything is -quick or standard mode with no Joern on the host, so each command takes minutes, not hours, on this 7 GB host. - -**Environment (instrument first).** - -- Run from a `git archive HEAD` export in the scratchpad (the source is frozen, per the long-runs rule). -- `uv sync --frozen --extra semantic`, which drops the installed `harnessx`. It is installed in `.venv` today. -- `OPENULTRASAST_SKIP_DOTENV=1` and `env -u DEEPSEEK_API_KEY -u OPENROUTER_API_KEY -u ANTHROPIC_API_KEY -u OPENAI_API_KEY`, - plus an empty `--config`, so no command can reach a model. - -The harness asserts `find_spec("harnessx") is None`, `find_spec("tree_sitter") is not None`, and a -`uv pip freeze` hash. It also prints the byte size of each input manifest, and fails on any non-zero exit code. - -**Commands** (one script, `benchmarks/harnessx_removal_equality.py record|compare `): - -1. `pytest -q -p no:cacheprovider --junitxml=suite.xml`, which gives outcome per test id. -2. `ousast pairs --json > pairs.json`. Instrument check: pair count > 0 and at least one TP. -3. `ousast benchmark benchmarks/manifests/python-vulnerable.toml --mode quick`. Keep - `benchmark_result.json`, `calibration_records.json` and `external_baseline_deltas.json`. Instrument check: - `expected > 0`. -4. `ousast scan benchmarks/fixtures/python-vulnerable --mode standard --config empty.toml`. Keep - `findings.json`, `verification.json`, `fusion.json`, `report.sarif`, `harness.json` and `manifest.json`. -5. `ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml --dry-run --no-pair-gate`, stdout plus - the dry-run journal. - -**Stored.** `benchmarks/measurements/-harnessx-removal-baseline/`: the normalized outputs, and `env.json` -(commit, freeze hash, instrument checks). The script's exclusion list is written into that directory's -`README` line. - -**Comparison.** JSON is parsed, normalized and re-serialized with sorted keys. Everything else is compared as -raw bytes. Excluded: - -- keys `runtime_seconds`, `scan_id`, `timestamp`, `started`, `finished`, `created_at`, `generated_at`, and any - key ending `_seconds` or `_ms` -- run directory names (timestamped, `benchmark.py:186`) -- `trace/events.jsonl` as a whole (timings) -- the export's absolute root path, replaced by `` - -For the suite, every test id that survives must keep its outcome. Deleted and renamed ids are compared through -the map in the Tests table above, and any other difference is a failure. Any difference in 2-5 blocks the merge. - -### 4. Memory layer (Requirement 6.1, 6.4) - -**Store** (`plane/memory.py`). The store lives at `results_root()/memory/` (`~/ousast-results/plane/memory/`, -honouring `OUSAST_RESULTS`, `reconciler.py:58-59`). It is JSON files, not SQLite: the rest of the plane is JSON, -diffable and greppable, and one run at a time holds the run lock, so SQLite's concurrency buys nothing. - -``` -memory/ - index.jsonl one row per ingested (run, task): sha256 of its memory.jsonl, row count - facts/.json facts.json by content (identical facts stored once) - repos/____/.jsonl the rows of one repository + 40-hex pin -``` - -Ingest is idempotent: an `index.jsonl` hit skips the delivery, and rows carry a deterministic `id`. Ingest also -refuses to write when `shutil.disk_usage` reports less than 1 GiB free. The disk is at 98% today, so it names -the store path in the error. - -**Storage backends (maintainer, 2026-09-30: asked to use the object-store features of the S3 server he runs, -including its JSON support).** The store is written against one interface, `MemoryStore` (`put_row`, `put_facts`, `get_facts`, -`rows(repo=..., pin=..., kind=..., where=...)`, `index`), with two backends of identical behaviour: - -- `FileStore`: the layout above, for this laptop and the unit tests. -- `S3Store`: an S3-compatible bucket (boto3; RustFS is the tested server), and the store of record once the plane runs on a separate - Kubernetes cluster. The layout maps onto one bucket with the same key paths: - - **Object metadata and tags** on every row object: `repo`, `pin`, `kind`, `family`, `run`, `population`, - `split`. Rules list by prefix (`repos//`) and filter by tag instead of scanning the bucket. - - **Server-side JSON queries**: `rows(where=...)` pushes the filter down with S3 Select - (`SelectObjectContent`, JSON Lines input, JSON output), so a proposal rule reads matching rows, not whole - files. Support is probed once per process against the server (a small Select on a known object); when the - server lacks it, the backend fetches and filters locally, logs the - fallback once, and returns the same rows. A query never silently returns nothing because a feature is - missing. - - **Versioning** on the bucket: each proposal's provenance records the object key and version id of every - memory row it cites, so its evidence cannot change underneath it (Req 6.4). Object lock (governance mode) is - optional for the `proposals/` prefix. - - **Lifecycle rules**: raw run outputs under `runs/` expire after a configurable number of days; `repos/`, - `facts/` and `proposals/` are kept. The near-full laptop disk stops being the store's limit. - - **Presigned URLs** (the step that removes the host receiver for the Kubernetes move, not built in this - spec): the reconciler can hand a task presigned PUT URLs for its outputs and GET URLs for its inputs in the - start request, so tasks write to the bucket directly and never hold credentials. The interface keeps - `presign_put`/`presign_get` so that follow-on changes the reconciler, not the store. - - **Bucket notifications** (follow-on): an event on new `repos/` rows can trigger the loop Run - (event-driven loop engineering). -- **Configuration**: `OUSAST_MEMORY` selects the backend (`file:///path` or `s3://[/]`); the S3 - endpoint and credentials come from `.env` (`S3_ENDPOINT`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, - optional `S3_REGION`), never from a manifest, never printed. Client: boto3 as an optional extra - (`openultrasast[s3]`); the core install stays PyYAML-only. -- **Tests**: the same contract tests run against `FileStore` and against `S3Store` pointed at an S3-compatible - endpoint (skipped unless `OUSAST_MEMORY_TEST_S3` is set), including the Select probe with its fallback - forced, versioned provenance, and metadata filters. The disk-space refusal applies to `FileStore` only. - -**`remember` task** (`plane/tasks/remember.py`, no Model, budget `{usd: 0, calls: 0}`, one per case). It reads the -case's delivered artifacts as inputs: `facts.json`, the `units.jsonl` of `va`/`vb`/`vc`, `final/agreed.json` and -`final/disputed.json` with the per-candidate rows, and `alerts.jsonl`. It takes the repository URL and pin from -the bound Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`) and the population from the Run annotation -`openultrasast.io/population`. It writes `memory.jsonl` and passes `facts.json` through. Task code interprets -the artifacts, so the reconciler and runner stay content-blind (ai-service-plane Req 3.4). After `reconciler.run` -returns, `cli._plane` calls `memory.ingest(run_dir)`. `ousast plane remember ` does the same by hand. - -**Row kinds** (every row has `id`, `kind`, `repo`, `pin`, `run`, `task`, `population`, `split`, `image`): - -- `facts`: `sha256`, `candidates_digest` (`repo_facts._digest`, `repo_facts.py:188`), `files`. -- `verdict`: `candidate` (`path::function`), `family`, `site`, `passes {a, b, c}` (true/false/null), `final` - (`agreed | rejected | disputed`), `tiebreak` (bool), `site_match` (bool/null), `usd`, `turns`. -- `unit_cost`: `pass`, `path`, `usd`, `calls`, `usage`, `turns`, `model`. -- `alert`: `rule_id`, `rule_status` (`enabled | shadow`), `path`, `line`, `function`, `pin_role` - (`vulnerable | fixed`). -- `proposal_outcome`: `edit_key`, `evidence_digest`, `outcome` (`accepted | reverted | rejected`), `gate_run`. - -**Fact reuse.** A facts entry is valid for the key (repo, pin, candidates digest, runner image digest). The image -is in the key because the host's source and the image's source can differ, and facts computed by other code are -not the same facts. `ousast plane workspaces --validation-set` writes the key as the Task annotation -`openultrasast.io/memory-key`. Annotations are already stripped before ax (`manifests.py:4-6`). Before -`reconciler.run`, `cli._plane` calls `memory.seed(run_manifest)`. For each Run entry whose Task carries a key with -stored facts and no state yet, seed writes `//facts.json` and a `summary.json` -(`status: done, units_done == units_total == 1, reused: {"sha256", "from_run"}`). It then calls -`reconciler.mark_done(run, task, reused=...)`, a public helper of about 10 lines that takes the run lock, sets -the state and releases it. The reconciler already skips `done` tasks (`reconciler.py:342`). The receiver already -serves any regular file under the run directory as a declared input (`egress.py:242-246`). A changed pin, a -changed candidate set or a rebuilt image recomputes. - -### 5. Proposals from memory (Requirement 6.2, 6.4) - -**Interface.** The proposal surface HarnessX would have used is `run_round` (`improve/evolve.py:149-247`). It -builds `RuleStatusEdit`s (`improve/validator.py:36-47`), filters them through the journal's novelty gate -(`journal.reverted_edit_keys`, `journal.py:26-38`), validates each with `EvolveValidator.validate` -(`validator.py:75-81`, which accepts only `RuleStatusEdit` and `PolicyConstantEdit`), smoke-replays and gates. - -`improve/memory.py` exposes: - -``` -propose_from_memory(rows, ruleset_by_id, current_ledger, journal_rounds, *, excluded) -> list[MemoryProposal] -MemoryProposal = (edit: RuleStatusEdit, provenance: dict) -``` - -`run_round` gains `proposals: Sequence[MemoryProposal] = ()`. It appends their edits after the benchmark's edits -and keeps the first edit per `key()`. The validator and the gate are untouched. `ousast improve` gains -`--memory [DIR]` (off by default, so the deterministic `improve` output is byte-identical) and -`--qualify-population NAME` (repeatable). - -**Rules** (the thresholds are module constants and part of the provenance record). Join key: an `alert` and a -`verdict` match when repo, pin and `path::function` are equal. A `disputed` final counts as no evidence in -either direction. - -- **M1, repeated false alerts** (`enabled -> shadow`). Rule R qualifies when both hold: - - at least 3 R alerts sit at candidates whose final is `rejected`, or on a `fixed` pin inside the case's declared - fix range, across at least 2 distinct repositories; - - no R alert sits at an `agreed` candidate or a `site_match` site. -- **M2, missed declared sites** (`shadow -> enabled`). Shadow rule R qualifies when all three hold: - - its shadow alerts hit at least 2 `agreed` candidates with `site_match == true`, across at least 2 - repositories; - - no enabled rule alerts at those candidates; - - R has no alert on a `fixed` pin at the same function and at most 1 alert at a `rejected` candidate. -- **M3, frequently disputed families.** Families with at least 5 candidates and a dispute rate of at least 0.3 go - to `signals.json` as advisory input for plane routing (for example, scheduling pass `c` up front). They are not - an edit and never reach the validator. - -**Provenance.** Each proposal's `rationale` is `memory:`, where `proposal_id` = sha256 of -(rule, edit key, sorted evidence row ids, thresholds). The sidecar `memory_proposals.jsonl`, next to the journal, -holds: - -- `proposal_id`, `edit_key`, `rule_id` -- `evidence` (the row ids plus their repo, pin, run and task) -- the counts that met each threshold -- `excluded` (populations and pins removed by the guard, with counts) -- the store's `index.jsonl` digest at proposal time - -`evolve._record` (`evolve.py:366-396`) adds `"evidence": proposal_id` to an edit's journal entry only when the -edit came from memory. Otherwise the journal is unchanged byte for byte. A reader follows journal, then sidecar, -then store rows, then the run directory. - -**Novelty.** `journal.reverted_edit_keys` does not block a rule edit that carries a rationale (`journal.py:36`), so -a memory proposal would otherwise come back every round. The proposer itself skips any key whose reverted journal -entry has the same evidence digest. New evidence is the "new rationale" that allows a retry. - -**Train-on-test guard.** A row is excluded from proposals when any of these holds: - -- (a) its `population` is one of `--qualify-population`, which is also where the proposal will be gated or - qualified; -- (b) its (repo, pin) is a case of the benchmark manifest being gated; -- (c) its (repo, pin) is a `split = "holdout"` pair of the pair catalog, which the per-profile holdout clause - (`evolve.py:204-214`) measures. - -This is the rule the closed-loop leak broke, where a holdout pair taught the shape that "recovered" it. Here the -filter runs before any rule sees a row. Concretely, rows from `validation-46` (population -`independent-v2/validation`) never propose edits that are then qualified on `independent-v2/validation`. The -tie-break was designed on that set, so it is development evidence only. - -### 6. The loop as a plane Run (Requirement 6.3) - -`ousast plane workspaces --validation-set ... --loop` extends the generated Run. It also emits a Workspace for each -case's fixed pin (the population's `fixed` sha, already read by `generate.fix_ranges`, `generate.py:89`): - -| Task (per case unless noted) | Model | Budget | Inputs -> outputs | -| --- | --- | --- | --- | -| `-facts`, `-va`, `-vb`, `-agree`, `-vc`, `-final` | verify passes: `deepseek-flash`; others none | as generated today (`generate.verify_budget`) | unchanged | -| `-alerts` (new) | none | `{0, 0}`, `serialize: engine` | vulnerable + fixed Workspaces -> `alerts.jsonl` (quick-mode `quick_scan_findings` with enabled and shadow rules, function-attributed) | -| `-remember` (new) | none | `{0, 0}` | facts, va, vb, vc, final, alerts -> `memory.jsonl`, `facts.json` | -| `loop-measure` (once) | none | `{0, 0}` | every `remember` + `memory-snapshot/rows.jsonl` -> `measure.json` (the increment metrics: declared sites agreed, cost per candidate, dispute rates) | -| `loop-propose` (once) | none | `{0, 0}` | measure, snapshot, journal -> `proposals.jsonl`, `memory_proposals.jsonl`, `signals.json` | -| `loop-improve` (once) | none | `{0, 0}`, `serialize: engine` | proposals; Workspace = this repository at the commit under test -> `gate.json` (the `RoundOutcome`), `journal.json`, `rule_policy.json` (only when accepted) | - -Before the Run starts, `memory-snapshot` is seeded into the run directory like reused facts. It holds the store -rows after the guard, since a sandboxed task cannot read the host store. A `{usd: 0, calls: 0}` budget makes -`budget.py` refuse any call, so a model-free task that tries one fails loudly rather than spending. Attribution -per task is the existing `ousast plane status` table. - -**The gate's verdict** lands in `~/ousast-results/plane//loop-improve/gate.json` and is ingested as -`proposal_outcome` rows. An accepted `rule_policy.json` is adopted into the repository only by a maintainer -commit, as any ledger change is today (README "Rule-level loop"). - -### 7. Documentation and specs (Requirement 4) - -- **README.** Replace "Configuring the HarnessX agentic plane" (README.md:293-329) with "The agentic plane (ax)": - what runs there (repo-facts, verify, agree, memory), `ousast plane run|status|remember`, a pointer to - `ops/ax/README.md`, and one tagged line saying HarnessX was retired. Rename "The HarnessX self-improving cycle" - (:330) to "The self-improving cycle". Its calibration ledger and `ousast improve` content is not HarnessX and - stays. The MetaAgent paragraph (:393-400) is replaced by the memory proposer (`--memory`, provenance, guard). - Drop the `panel_model`/`decider_model` lines at :227-228. -- **`docs/examples.md` section 6** becomes "Running the agentic plane", with the plane doctor, run and status - commands. -- **`docs/threat-model.md:17-20, 32-34`**: egress is the per-task EgressPolicy (receiver, Git hosts, Model - `egress-hosts`, passthrough only). Credentials travel only in the start request. Spend is the per-task - `usd`/`calls` budget in the Run. -- **Retirement notes.** A dated note ("Retired 2026-..: HarnessX was removed by `harnessx-removal`; the agentic - plane is ax (`ai-service-plane`), proposals come from the plane memory. History below is unchanged.") goes at - the top of the requirements (and design, where present) of `harnessx-self-improving-rulesets`, - `learning-harness`, `three-stage-scan` and `openrouter-sast-harness`, and in `.kiro/steering/overview.md` near - :175. The other specs that mention HarnessX in passing keep their history untouched. -- `RELEASE_NOTES.md` gets one entry listing the retired keys. Kiro tooling stays out of user docs. - -## Data Models - -- `memory.jsonl` (remember output) and `repos//.jsonl`: the rows above, one JSON object per line, - sorted keys, stable `id` = sha256(kind, run, task, candidate|path|rule_id+line). -- `index.jsonl`: `{"run", "task", "sha256", "rows", "ingested"}`. -- `memory_proposals.jsonl`: `{"proposal_id", "edit_key", "rule_id", "rule": "M1|M2", "evidence": [...], - "counts": {...}, "thresholds": {...}, "excluded": {...}, "store_digest"}`. -- Journal edit entry: the existing fields (`evolve.py:381-384`) plus `"evidence"` only for memory edits. -- Equality record: `env.json` `{"commit", "freeze_sha256", "harnessx_importable": false, "tree_sitter": true, - "inputs": {path: bytes}, "exclusions": [...]}`. - -## Error Handling - -- `RetiredConfigError`: exit 2 with the key and its replacement, before any stage runs. -- A `[harnessx]` section produces exactly one warning, and loading continues. -- `memory.ingest` on a run without `remember` outputs is a no-op that says so. A malformed row fails the ingest - and names the file and line, and nothing partial is written: rows are staged in a temporary file and renamed. -- A low-disk refusal names the store and the free space. The plane run's result is unaffected, and ingest can be - repeated with `ousast plane remember`. -- `seed` never overwrites an existing task directory or a non-pending state. A facts file whose sha256 no longer - matches is dropped from the store with a warning, and that task recomputes. -- `propose_from_memory` with an empty store after the guard returns no proposals, and `run_round` reports - `no_proposals` as today. -- The equality script exits non-zero on the first failed instrument check. It also exits non-zero on any - difference outside the exclusion list and prints the first differing path and key. - -## Testing Strategy - -- `tests/test_config.py`: the one-warning section and the retired keys (above). -- `tests/test_no_harnessx_references.py`: the tagged-only search of Requirement 2.4. -- `tests/test_plane_memory.py`: ingest is idempotent and content-addresses identical facts. `seed` marks a facts - task done and serves its file, and does not for a changed pin, candidate digest or image. Low-disk refusal - (monkeypatched `disk_usage`). -- `tests/test_plane_remember.py`, `test_plane_alerts.py`, `test_plane_loop.py`: fixture run directories built from - the recorded validation-46 shapes. `remember` rows match `agree`'s per-candidate rows. `alerts` attributes - functions. `measure` reproduces the increment's recorded metrics on the recorded artifacts. `propose` and - `improve` work end to end on a fixture manifest. -- `tests/test_improve_memory.py`: - - M1 and M2 fire at their thresholds and not below. - - A disputed candidate counts neither way. - - The guard removes rows by population, manifest case and holdout pair, so a proposal that exists only because - of a holdout row disappears. - - Provenance ids are stable across runs. - - A reverted memory edit is not re-proposed on the same evidence, and is re-proposed on new evidence. - - `run_round` without proposals writes a journal byte-identical to today's. -- `tests/test_plane_reconciler.py`: `mark_done` under the lock. The existing under-500-lines test stays. -- The existing gate and pair tests, `test_benchmark_output_is_byte_identical_to_golden_baseline` - (`test_gate.py:104`) and the equality script (section 3) are the regression net. - -## Requirement Traceability - -| Requirement | Design section | Verified by | -| --- | --- | --- | -| 1.1 fate of every capability | 1 (table) | this document; commit 5 | -| 1.2 moved capabilities reachable, Model per task, fixture test | 1 (existing `verify`/`agree`), 4-6 (new tasks are Model-free) | `test_plane_verify.py`, `test_plane_tiebreak.py`, new plane task tests | -| 1.3 retired keys fail naming the replacement | 1 (`RetiredConfigError`) | `test_config.py` | -| 2.1 modules and guards gone | 2 Source | commit 6; audit test | -| 2.2 extra, override, lock gone; installs succeed | 2 Packaging | fresh-venv `pip install .` / `'.[semantic]'` | -| 2.3 tests deleted or renamed; audit regenerated | 2 Tests, Module audit | `test_module_audit.py` | -| 2.4 search returns only retirement notes | 2 Search | `test_no_harnessx_references.py` | -| 3.1 baseline without the extra | 3 | commit 1 record | -| 3.2 byte-identical after, exclusions written | 3 Comparison | commit 7 record | -| 3.3 `[harnessx]` loads with one warning | 1 | `test_config.py` | -| 4.1-4.3 docs, retirement notes, Kiro out of user docs | 7 | commit 8 review | -| 5.1-5.2 order, green steps, no deletion before replacement | Commit sequence | per-commit gates | -| 6.1 store keyed by repo + pin, fact reuse | 4 | `test_plane_memory.py`, `test_plane_remember.py` | -| 6.2 deterministic proposals through validator and gate | 5 | `test_improve_memory.py` | -| 6.3 loop as a plane Run with per-task Models and budgets | 6 | `test_plane_loop.py`; `ousast plane status` attribution | -| 6.4 provenance and train-on-test guard | 5 | `test_improve_memory.py` | - -## Commit sequence (Requirement 5), mapped to future tasks - -Each commit is gated on the full suite's own exit code, `ruff` and `mypy`. - -1. **Baseline** (3.1): the equality script and `benchmarks/measurements/-harnessx-removal-baseline/`, - recorded with the extra uninstalled. -2. **Memory store and `remember`** (6.1): `plane/memory.py`, `tasks/remember.py`, `reconciler.mark_done`, seed and - ingest wiring in `cli._plane`, generator annotations, fixture tests. -3. **Memory proposer** (6.2, 6.4): `improve/memory.py`, `run_round(proposals=...)`, `ousast improve --memory`, - sidecar and journal evidence, guard tests. -4. **Loop Run** (6.3): `alerts`, `measure`, `propose` and `improve` tasks, `--loop` generation, fixed-pin - Workspaces, fixture tests. -5. **Retire the scan paths** (1, 3.3): the `cli.py`, `fusion.py`, `verify_judge.py` and `stage_processors.py` - changes, `RetiredConfigError`, the one warning, and the test renames that follow. At this point nothing imports - `harness_ext`/`hunter_harness`. Their replacements (plane tasks, and the memory proposer from 2-4) have landed. -6. **Delete code and packaging** (2): the two modules, the extra, the mypy override, `uv.lock`, the deleted tests, - the fixture rename, audit rows and the manifest regeneration, and the reference search test. -7. **Equality proof** (3.2): rerun the script, commit the comparison record next to the baseline, and link it in - the commit message. -8. **Docs and specs** (4): the README, `docs/examples.md`, `docs/threat-model.md`, retirement notes, steering and - release notes. - -## Risks - -- **Existing users of the LLM scan paths.** There is no `--llm` flag. Users who set `[models] verifier` or - `[fusion] panel_model` get a hard error, which is intended (1.3). Users who installed the extra and set - `[models] hunter` lose the second (HarnessX) hunter without an error. Only the release note covers that, - because the key is still valid. No measurement suggests either path found anything the remaining paths miss. - The absence of a measurement is itself the risk. -- **Plane precision is unsolved.** `plane-increment-2.json` agrees 28 candidates, of which 16 are declared - sites. The other 12 are unadjudicated and several may be false. M1 treats a `rejected` final as negative - evidence, while plane recall is 16/20, so "rejected" is not "safe". The thresholds (2 or more repositories, zero - agreed hits) and the unchanged recall/FP gate are the guard. Precision work (judge design) remains out of scope. -- **Evidence is thin.** Today's store would hold 15 cases from one validation population, which is exactly the - population the guard excludes for its own qualification. Expect `no_proposals` until a second population has - run through the loop. -- **Disk.** The home volume is 98% full (2.4 GB free). One validation-46-sized Run adds about 3 MB of facts - (content-addressed; about 0.1 MB when pins are unchanged) plus about 0.5 MB of rows. Ingest refuses below 1 GiB - free. The store has no eviction in this spec. -- **The baseline measures the environment as well as the code.** `.venv` has `harnessx` installed today. A - baseline taken without re-syncing would include it. The script's `find_spec` and freeze checks exist for that. -- **Seeding writes reconciler state from outside the reconciler.** It is confined to `mark_done` under the run - lock. The reconciler's line budget (465 of 500) leaves no room for fact reuse inside the reconciler. diff --git a/.kiro/specs/harnessx-removal/requirements.md b/.kiro/specs/harnessx-removal/requirements.md deleted file mode 100644 index 0b5b71e..0000000 --- a/.kiro/specs/harnessx-removal/requirements.md +++ /dev/null @@ -1,125 +0,0 @@ -# Requirements Document - -## Introduction - -HarnessX is the optional agentic plane (`openultrasast[harnessx]`): an LLM hunter pool, an LLM judge and LLM -fusion panels, each behind a capability guard with a deterministic fallback, plus a lazy bridge that hosts -deterministic stages under a HarnessX processor. The ai-service-plane increment met its gate on google/ax -(`benchmarks/independent/plane-increment-2.json`), which is the condition the maintainer set on 2026-09-29: -"goal is to remove harnessx and to shift to ax fully once we establish we have value." This feature removes -HarnessX from the product and moves every LLM capability it carried either onto the plane or into a recorded -retirement. Inventory: `brief.md` (verified 2026-09-30). - -Maintainer, 2026-09-30: "Please remove harnessx completely. the memory layer that saves tokens takes over the -evolutionary improvements as well as loop engineering." So nothing of HarnessX stays, not even as an optional -path, and the self-improving cycle HarnessX was meant to drive (the LLM `MetaAgent.evolve` proposer plugged into -`improve/evolve.py`) is carried by the plane's memory layer instead: what runs record (repository facts, -verdicts, disputes, costs) becomes the input to proposals and to the run-measure-propose-gate loop. - -## Boundary Context - -- **In scope**: the HarnessX-only modules (`harness_ext.py`, `hunter_harness.py`); the guarded branches in - `cli.py` (scan `--llm` path, `HxScanOrchestrator`), `verify_judge.py` (judge path), `fusion.py` (LLM panels), - `stage_processors.py` (`host_under_harnessx`), `config.py` (`HarnessxConfig`); the `harnessx` extra, its mypy - override and lock entry; the tests that exercise HarnessX; README and docs sections; dated retirement notes in - the specs and steering that name HarnessX. -- **Also in scope**: the plane memory layer as the source of evolutionary proposals and the loop (Requirement 6), - replacing the HarnessX proposer. -- **Out of scope**: changing the deterministic engine, rulesets or `evolve`'s validator and gate (proposals change - source, not how they are validated and gated); new detection logic; qualification on an independent population; the plane's own precision work (judge design) -- - where a capability moves to the plane, this spec wires it, it does not improve it. -- **Adjacent expectations**: every deterministic path (quick mode, `regress`, `benchmark`, `gate`, `evolve`) - produces byte-identical output before and after; `vulnerabilities-over-plumbing` applies, so capabilities are - kept or retired by what they detect, reported in TP/FP terms where a measurement exists. - -## Requirements - -### Requirement 1: HarnessX is removed completely; the plane is the only agentic path - -**User Story:** As the maintainer, I want no HarnessX path left, optional or not, and every LLM capability either -on the plane or explicitly retired, so that nothing is lost silently. - -#### Acceptance Criteria - -1. No HarnessX code path remains, optional or guarded. The design lists each capability -- LLM hunter pool - (`ousast scan --llm`), LLM judge (`verify_judge`), LLM fusion panels (`fusion`), deterministic stages hosted - under HarnessX (`host_under_harnessx`), the LLM evolution proposer -- with its fate: carried by a plane task - (named), carried by the memory layer (Requirement 6), or retired with the deterministic path kept. -2. A capability moved to the plane is reachable from the CLI through the plane (`ousast plane run` with a - documented Run, or a CLI flag that submits one), binds its own Model per task (ai-service-plane Req 7), and - has a fixture test like the other plane tasks. -3. A retired capability's CLI flags and config keys fail with a message that names the replacement or says it - was retired, never a silent no-op. - -### Requirement 2: HarnessX code, packaging and tests are gone - -**User Story:** As the maintainer, I want no HarnessX code path left to maintain. - -#### Acceptance Criteria - -1. `harness_ext.py` and `hunter_harness.py` are deleted; no module imports `harnessx`; `use_harnessx` - parameters, `has_harnessx` checks and `host_under_harnessx` are removed; the deterministic fallbacks become the - only code path where a capability is retired. -2. The `harnessx` extra, the mypy override for `harnessx.*` and the lock entry are removed; `pip install .` and - `pip install '.[all]'` (or whatever extras remain) succeed. -3. Tests that only exercise HarnessX are deleted; tests that assert fallback behaviour are kept and renamed to - describe the behaviour, not the absence of HarnessX; the module audit is regenerated with no orphan. -4. A search for `harnessx` (case-insensitive) in `src/`, `tests/`, `pyproject.toml` and the README returns only - the retirement notes of Requirement 4. - -### Requirement 3: Deterministic behaviour is unchanged, and proven so - -**User Story:** As the maintainer, I want evidence, not assurance, that removing HarnessX changed nothing a -deterministic run produces. - -#### Acceptance Criteria - -1. Before any deletion, a baseline is recorded without the extra installed: the full host suite, and the outputs - of `ousast regress` and one `ousast benchmark` replay on the development pins, stored under - `benchmarks/measurements/`. -2. After the removal the same commands produce byte-identical outputs (timestamps and durations excluded, and - the exclusion list is written down); any difference blocks the merge. -3. A config file that still carries a `[harnessx]` section loads with one warning that names the plane, and the - section is otherwise ignored. - -### Requirement 4: Documentation and specs tell the new story - -**User Story:** As a reader of the README or a spec, I want to learn that the agentic plane is ax, not HarnessX. - -#### Acceptance Criteria - -1. The README sections "Configuring the HarnessX agentic plane" and "The HarnessX self-improving cycle" are - replaced by a short section on the ax plane that points at `ops/ax/README.md`; `docs/examples.md` section 6 and - `docs/threat-model.md` (egress and spend caps) describe the plane's egress policy and budgets instead. -2. Specs that name HarnessX (`harnessx-self-improving-rulesets`, `learning-harness`, `three-stage-scan`, - `openrouter-sast-harness`) and `.kiro/steering/overview.md` get a dated retirement note at the top; their - history is not rewritten. -3. Maintainer-only Kiro tooling stays out of user docs (standing rule). - -### Requirement 5: The removal lands in one reviewable sequence - -**User Story:** As the maintainer, I want to review the removal in steps that each leave main green. - -#### Acceptance Criteria - -1. Order: baseline (3.1) -> capabilities moved or retired (1) -> code and packaging removed (2) -> equality proof - (3.2) -> docs and specs (4). Each step is a commit gated on the full suite's own exit code, ruff and mypy. -2. No step deletes code whose replacement (Requirement 1) has not landed and passed its fixture test. - -### Requirement 6: The memory layer carries evolutionary improvement and loop engineering - -**User Story:** As the maintainer, I want the token-saving memory layer to be what the system learns from, so -that improvement proposals and the improvement loop come from recorded evidence instead of a HarnessX agent. - -#### Acceptance Criteria - -1. Plane runs persist their memory beyond the run directory in one store keyed by repository and pin: repository - facts (`repo-facts`), per-candidate verdicts per pass with the final decision, disputes and tie-breaks, and cost - and tool turns per unit; later runs reuse facts for an unchanged pin instead of recomputing them. -2. `evolve`'s proposals are generated from that store (for example: families or rules whose candidates are often - disputed, missed declared sites, repeated false alerts), deterministically and without a HarnessX agent; they - pass through the existing validator and gate unchanged. -3. The improvement loop -- run, measure, propose, validate, gate -- is expressible as a plane Run whose tasks bind - their own Models and budgets (ai-service-plane Req 7), so each iteration's token spend is attributed per task. -4. A proposal records which memory entries it came from, so a later reader can trace every accepted change to the - evidence behind it; proposals never train on the population used to qualify them (the train-on-test rule). diff --git a/.kiro/specs/harnessx-removal/spec.json b/.kiro/specs/harnessx-removal/spec.json deleted file mode 100644 index f7c8cbf..0000000 --- a/.kiro/specs/harnessx-removal/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "feature_name": "harnessx-removal", - "created_at": "2026-09-30T08:00:00Z", - "updated_at": "2026-09-30T16:00:00Z", - "language": "en", - "phase": "implementation-complete", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true -} diff --git a/.kiro/specs/harnessx-removal/tasks.md b/.kiro/specs/harnessx-removal/tasks.md deleted file mode 100644 index 853604b..0000000 --- a/.kiro/specs/harnessx-removal/tasks.md +++ /dev/null @@ -1,62 +0,0 @@ -# Implementation Plan - -Order follows the design's commit sequence (Requirement 5): nothing is deleted before its replacement has landed -and passed its fixture test. Every task is a commit gated on the full suite's own exit code, `ruff check`, -`ruff format --check` and `mypy`; the reconciler stays under 500 lines. - -- [x] 1. Baseline without the extra - - Equality script (design §3) with its `find_spec("harnessx")` and freeze checks; re-sync `.venv` without the - `harnessx` extra first. - - Record `benchmarks/measurements/-harnessx-removal-baseline/`: full suite, `ousast pairs --json`, the - `python-vulnerable` quick benchmark, one standard-mode scan, `ousast improve --dry-run`; the exclusion list - written next to it. - - _Requirements: 3.1, 5.1_ - -- [x] 2. Memory store and the `remember` task - - `plane/memory.py`: the `MemoryStore` interface with `FileStore` and `S3Store` (object metadata/tags, - S3 Select JSON queries probed with a local fallback, versioned provenance, lifecycle, presign stubs; - `OUSAST_MEMORY` selects; S3 settings from `.env`; boto3 as an optional extra); JSON rows per - repository + pin, facts by content hash, `FileStore` ingest refuses below 1 GiB free, - `plane/tasks/remember.py` (model-free), `reconciler.mark_done` under the run lock, seed and ingest wiring in - `cli._plane`, generator annotations; fact reuse when repository, pin, candidate set and runner image match. - - Fixture tests: rows from a recorded Run, reuse on an unchanged pin, recompute on a changed one, disk refusal. - - _Requirements: 6.1, 6.4_ - -- [x] 3. Proposals from memory - - `improve/memory.py` with rules M1 (demote) and M2 (promote) and the advisory disputed-family signal; - `run_round(proposals=...)`; `ousast improve --memory` (opt-in; default output byte-identical); provenance per - proposal in the sidecar and journal; the train-on-test guard (qualification population, gated benchmark cases, - holdout pairs). - - Tests: each rule on crafted rows, the guard drops excluded rows, provenance round-trips, default `improve` - unchanged. - - _Requirements: 6.2, 6.4_ - -- [x] 4. The loop as a plane Run - - Tasks `alerts`, `measure`, `propose`, `improve` (no Model, zero budget); `--loop` Run generation; fixed-pin - Workspaces; the gate's verdict in `/loop-improve/gate.json`; adoption stays a maintainer commit. - - Fixture tests with the fake ax; one live loop Run on the local ax recorded (no model spend beyond the verify - passes it reuses from memory). - - _Requirements: 6.3_ - -- [x] 5. Retire the HarnessX scan paths - - `cli.py`, `fusion.py`, `verify_judge.py`, `stage_processors.py` changes; `RetiredConfigError` for - `[models] verifier`, `[fusion] panel_model`/`decider_model`; one warning for a `[harnessx]` section (the only - tagged retirement text allowed in `src`, enforced by a test); tests renamed to the behaviour they assert. - - _Requirements: 1.1, 1.2, 1.3, 3.3_ - -- [x] 6. Delete HarnessX code and packaging - - Delete `harness_ext.py`, `hunter_harness.py`, `verify_judge.py`'s judge path, the HarnessX-only tests; remove - the extra, the mypy override and the `uv.lock` entry; rename the conftest fixture; audit rows and manifest; the - reference search test (Req 2.4). - - _Requirements: 2.1, 2.2, 2.3, 2.4_ - -- [x] 7. Equality proof - - Rerun the baseline script; commit the comparison record next to the baseline; any difference outside the - exclusion list blocks the merge. - - _Requirements: 3.2_ - -- [x] 8. Documentation and specs - - README ax-plane section pointing at `ops/ax/README.md`; `docs/examples.md` §6 and `docs/threat-model.md` - (egress policy, budgets); dated retirement notes in the four specs and `.kiro/steering/overview.md`; release - note naming the silent loss of the second hunter for users who set `[models] hunter`. - - _Requirements: 4.1, 4.2, 4.3_ diff --git a/.kiro/specs/harnessx-self-improving-rulesets/blueprint-alignment.md b/.kiro/specs/harnessx-self-improving-rulesets/blueprint-alignment.md deleted file mode 100644 index 58ca378..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/blueprint-alignment.md +++ /dev/null @@ -1,96 +0,0 @@ -# DocuHarnessX ↔ OpenUltraSAST — Blueprint Alignment - -How OpenUltraSAST's HarnessX integration (Phase 3+) aligns with **DocuHarnessX** -(`/home/mc/Source/DocuHarnessX`), the author's other HarnessX project. Both are HarnessX -bundles steered from claude-code/opencode via Kiro spec-driven development. Goal: one shared -architecture the steering layer drives identically. - -## Orientation - -DocuHarnessX is **agent-first** — an 8-stage pipeline (`ingest → analyze → classify → plan → -write → review → assemble → deploy`) that *is* the agent loop. OpenUltraSAST is a -**deterministic-pipeline + agentic-plane hybrid** — a sync `HarnessRuntime` driving 13 stages, -with HarnessX confined to the one stage that is genuinely an agent loop (the hunter pool). They -converge on the same load-bearing patterns and diverge only where SAST's determinism and -supply-chain posture require it. - -**What "self-improving multi-agent system" means in DocuHarnessX (important):** it does **not** -use `meta_harness` / `MetaAgent.evolve` at all (the `Train` loop is explicit future / out of -scope). Its self-improvement is realized two ways: (1) an in-product **fail-closed LLM-judge -quality firewall** (COBESY: fixed criteria, threshold-re-derived verdicts, unavailable → fail), -and (2) the **claude-code/opencode Kiro steering loop** (fresh implementer → adversarial -reviewer → debugger subagents per task). OpenUltraSAST inherits both substrates and *additionally* -builds the `MetaAgent.evolve` loop DocuHarnessX leaves unbuilt. - -## Shared blueprint (the spine) - -| Concern | DocuHarnessX | OpenUltraSAST | Verdict | -|---|---|---|---| -| Composition seam | `make_docgen` in `docuharnessx/bundle.py` — the only HarnessX import site; `control \| stages_builder()`; returns model-less `HarnessConfig` | `build_sast()` in `harness/build_sast.py` (mirrors `examples/coding/build_coding`), lazy behind `_has_harnessx()` | **shared** | -| Model binding | CLI `prepare_run()`: `model_config.agentic(make_docgen(...))`; model on `harness.model_config.main`, never in config | `HxScanOrchestrator`: `asyncio.run(model_config.agentic(cfg).run(ScanTask(...)))` | **shared** | -| Processor base | `NoOpStage(MultiHookProcessor)`, content-neutral on `step_end`, real module-level classes for `_target_` resolution | `MultiHookProcessor(skip_model=True)` + `SlotContractMixin`, real module-level classes | **shared** | -| Inter-stage seam | `RunContext` over append-only slot constants; frozen versioned handoff dataclasses | typed `sast.*` slots + `SlotContractMixin` (`reads_slots`/`writes_slots`) | **shared** | -| Domain / "ontology" seam | `.docuharnessx/ontology.yaml` → `Vocabulary` (Role×Subject×Intent), loaded at run start | central **CWE policy + ruleset** (`policy/CWE_Score.tsv` + `ruleset/*.toml` + `rule_policy.json`), loaded at run start | **analogous** | -| Acceptance firewall | COBESY all-of gate; judge `passed` untrusted/re-derived; unavailable → fail | `VerifyProcessor` (llm-judge + structural fallback, rejects unknown reachability, fail-closed) | **analogous** | -| Steering | Kiro SDD; `CLAUDE.md`; `.kiro/{steering,specs}`; `kiro-impl` fresh-subagent + adversarial review; `kiro-verify-completion` | same Kiro convention; `.kiro/specs/harnessx-self-improving-rulesets/*`; OpenCode steers, kiro-* maintainer-only | **shared** | -| Self-improvement (`evolve`) | **none** — future seam | `improve/evolve.py → MetaAgent.evolve`, bounded `rule`/`policy` levers, `rule_policy.json` ledger | **diverges (OUS builds it)** | -| Dependency posture | `harnessx` direct git dep + pyyaml | core zero-dep; `harnessx` optional SHA-pinned extra, lazy | **diverges (by design)** | - -## Three principled divergences (and reconciliation) - -1. **Agent-first vs hybrid.** OUS keeps the sync `HarnessRuntime` through Phases 0–4 and confines - HarnessX to the hunter pool (the runtimes are "incompatible at every load-bearing point — - bridge, do not map"). Reconcile by treating `build_sast()` as building **only the hunter - sub-harness**, applying DocuHarnessX's `make_docgen` pattern *at the hunter-pool boundary*. - Optional Phase 5 converges OUS toward DocuHarnessX's agent-first end-state (retire - `HarnessRuntime`) — the divergence is temporal, and DocuHarnessX is the north star. -2. **Editable ontology vs authoritative severity.** DocuHarnessX's whole vocabulary is editable - presets. OUS splits its domain seam into a **mutable detection vocabulary** (`ruleset/*.toml`, - loop- and human-editable) and an **immutable severity/scope authority** (`CWE_Score.tsv`, - verycode-upstream, the loop must never edit). Where DocuHarnessX relies on convention ("never a - hardcoded literal"), OUS relies on a **validator** (`EvolveValidator._policy` forbids editing the - 0–5 severity) because a loop, not just a human, edits it. -3. **Firewall-as-reward vs reward/feasibility split.** DocuHarnessX's firewall *is* the signal - (binary pass/fail). OUS must feed `MetaAgent.evolve`, so it keeps a **scalar reward** - (`project_score`) **separate from** a **hard feasibility constraint** (≥90% recall / <10% FP), - un-folded so the loop can't game the number. Reuse DocuHarnessX's fail-closed firewall verbatim - at the *per-finding* `VerifyProcessor` granularity; layer the reward/feasibility split on top. - -## Phase 3 blueprint for OpenUltraSAST (mirror DocuHarnessX) - -- **`build_sast()` factory** = the single HarnessX composition import site (DocuHarnessX's "blast - radius is this file" rule on `bundle.py`); `control | hunter_stages_builder()`; raised loop - thresholds (large-repo scanning revisits shapes, like large-repo docs); model bound later in - `HxScanOrchestrator`. OUS delta: the `_has_harnessx()` guard + raise-then-fallback to - `run_hunter_pool()`. -- **Ontology seam** = CWE policy + ruleset as OUS's "ontology": `load_ruleset(path, ledger)` + - `load_policy(tsv)` loaded into `sast.*` slots at run start; `resolve_severity()` is the single - severity authority (mirrors DocuHarnessX's "every transform reads the loaded store, never a - literal"); absent ledger → byte-identical baseline; unmapped enabled-rule CWE → fail-loud - (stricter than DocuHarnessX, because severity is authoritative). -- **Processor set**: register HarnessX classes directly **only inside the hunter sub-harness** - (LoopDetection, CostGuard/TokenBudget, ToolFailureGuard, ToolFilter, SelfVerify, EpisodeMetrics/ - OTel/Checkpoint); everything deterministic is a `skip_model=True` `MultiHookProcessor` over slots - (SarifIngest/Preprocess/EntryPoint/Rank/Calibrate/QuickFindings/PolicyScoring/Verify/Score) with - stable `make__processor()` factories so a spec replaces exactly one stub in place. -- **`MetaAgent.evolve` loop** (Phase 4, on the same substrate): two bounded levers (`rule`, - `policy`); reward = `project_score`; feasibility = recall/FP gate (un-folded); signal path - benchmark per-rule precision/recall + `rule_policy_delta` → `rule_signals.json` → - `MetaAgent.evolve`; gate = validity (canonicalize → contract → `_rules`/`_policy` → replay) → - novelty/evidence → re-benchmark → attribution → accept or **byte-for-byte revert** (the - structural analog of DocuHarnessX's `DEFAULT_UNAVAILABLE_VERDICT="fail"`); ledger = - `rule_policy.json` (OUS's resume-safe analog of DocuHarnessX's markdown round journal). -- **Steering surface**: reuse DocuHarnessX's `kiro-*` skills verbatim (maintainer-only); add - SAST-domain `.kiro/steering/` rules pinning the detection gate as a hard invariant, the zero-dep - guard, and the `pip --dry-run` audit gate; traceable per-module docstrings citing spec Req/Decision - numbers. - -## Net - -The two projects already share the spine — one composition factory, model bound in the CLI via -`ModelConfig.agentic`, content-neutral processors over typed slots, a project-loaded domain seam, -a fail-closed acceptance firewall, and the Kiro fresh-subagent steering loop. OpenUltraSAST diverges -in exactly three principled places (hybrid runtime confining HarnessX to the hunter pool; it *builds* -the `MetaAgent.evolve` loop with a reward/feasibility split + `rule_policy.json` ledger; zero-dep core -with HarnessX opt-in). Adopting DocuHarnessX's load-bearing patterns at the points above gives the -two projects one architecture that opencode/claude-code steers identically. diff --git a/.kiro/specs/harnessx-self-improving-rulesets/dependency-audit.md b/.kiro/specs/harnessx-self-improving-rulesets/dependency-audit.md deleted file mode 100644 index 917e2cb..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/dependency-audit.md +++ /dev/null @@ -1,116 +0,0 @@ -# HarnessX Dependency Dry-Run Audit (Task 5.1) - -Recorded realized transitive dependency graph and the accept-vs-vendor decision for -adopting HarnessX (`git+https://github.com/Darwin-Agent/HarnessX.git`) in Phase 3. - -## Method - -```bash -uv venv /tmp/hx-audit --python 3.11 -uv pip install --dry-run --python /tmp/hx-audit/bin/python -``` - -- **118 transitive packages** resolved from HarnessX's **30 direct** `[project.dependencies]` - (all mandatory upstream; the only extra is `dev`). -- Pinned versions observed: `harnessx 0.1.0`, and engine image `registry.dagger.io`/SDK - unrelated; LLM SDKs `anthropic 0.111.0`, `openai 2.43.0`, `litellm 1.89.3`. - -### Realized transitive graph (118) - -``` -aiofiles aiohappyeyeballs aiohttp aiosignal annotated-doc annotated-types anthropic -antlr4-python3-runtime anyio attrs bracex certifi cffi charset-normalizer click cryptography -distro docker dockerfile-parse docstring-parser e2b et-xmlfile fastapi fastuuid filelock -frozenlist fsspec googleapis-common-protos greenlet grpcio h11 h2 harnessx hf-xet hpack -html2text httpcore httptools httpx httpx-sse huggingface-hub hydra-core hyperframe idna -importlib-metadata jinja2 jiter jsonschema jsonschema-specifications litellm loguru lxml -markdown-it-py markupsafe mcp mdurl multidict omegaconf openai openpyxl opentelemetry-api -opentelemetry-exporter-otlp opentelemetry-exporter-otlp-proto-common -opentelemetry-exporter-otlp-proto-grpc opentelemetry-exporter-otlp-proto-http opentelemetry-proto -opentelemetry-sdk opentelemetry-semantic-conventions packaging pdfminer-six pdfplumber pillow -playwright prompt-toolkit propcache protobuf pycparser pydantic pydantic-core pydantic-settings -pyee pygments pyjwt pypdfium2 python-dateutil python-docx python-dotenv python-multipart -python-pptx pyyaml referencing regex reportlab requests rich rpds-py shellingham six sniffio -sse-starlette starlette structlog tiktoken tokenizers tqdm typer typing-extensions -typing-inspection urllib3 uvicorn uvloop watchfiles wcmatch wcwidth websockets xlsxwriter yarl zipp -``` - -## Key finding: heavy backends are lazy; ~half the tree is dead weight - -Every heavy backend is **lazily imported** — `docker`/`e2b` are `TYPE_CHECKING`-guarded and -imported inside provider factories; `playwright` is imported inside `_get_local_page()`; -office/PDF libs inside `tools/builtin/read.py` parse functions; `sandbox/__init__.py` registers -docker/e2b via `try/except`. The `gateway/` is a **separate distribution**. So the 118-package -tree is an **install-time** cost; at **runtime only what a given agent path touches is loaded**. - -This is the answer to "I don't need most of these": neither OpenUltraSAST (SAST) nor the -author's DocuHarnessX (docs) loads the bulk of it — HarnessX simply declares every capability as -a *mandatory* dependency. - -| Category | Packages | SAST verdict | -|---|---|---| -| LLM SDKs | anthropic, openai, litellm, tiktoken, tokenizers, huggingface-hub, hf-xet, jiter, regex, tqdm, fsspec | **needed** | -| HTTP / async / pydantic core | httpx(+core/h11/h2/hpack/hyperframe), anyio, aiohttp(+glue), pydantic(+core/settings), certifi, urllib3, requests, sniffio | **needed** | -| Config / CLI / logging | hydra-core, omegaconf, antlr4-runtime, typer, click, rich, prompt-toolkit, structlog, loguru, jsonschema(+glue) | **needed** | -| Browser automation | **playwright, pyee, greenlet, html2text** | **dead weight** (+ out-of-band Chromium binary download via `playwright install`) | -| Sandbox backends | docker, dockerfile-parse, **e2b** | docker = useful (isolate hunter); **e2b = dead weight** | -| Web gateway / Lab UI | **fastapi, uvicorn, starlette, sse-starlette, httptools, websockets, uvloop, watchfiles, python-multipart, aiofiles** | **dead weight** (server surface; never started by a headless embed) | -| gRPC + OpenTelemetry | grpcio, protobuf, googleapis-common-protos, opentelemetry-* (8) | **optional** (tracing only) | -| Office / PDF | **python-pptx, python-docx, openpyxl, reportlab, pdfplumber, pdfminer-six, pypdfium2, pillow, xlsxwriter, lxml, et-xmlfile** | **dead weight** (SAST reads source, not documents) | -| MCP | mcp, httpx-sse | optional (only if the agent calls MCP tools) | - -Roughly **55–60 packages are opt-in capability weight** pulled in mandatorily, including a -browser binary and a Docker SDK — an awkward footprint for a supply-chain auditing tool. - -## Decision - -**ACCEPT HarnessX as the optional, SHA-pinned extra `openultrasast[harnessx]`, lazy-imported -behind `_has_harnessx()`. Keep a pre-specified vendored lean subset as a documented, triggered -fallback. Pursue upstream dependency-extras as the preferred long-term slimming path.** - -Rationale: -1. **Footprint is fully opt-in.** Core `dependencies = []` stays zero-dep; the governance / - scoring / benchmark / detection planes never pull HarnessX. Only `openultrasast[harnessx]` - installs the tree, and lazy imports keep heavy backends cold unless the path runs. -2. **Author precedent.** DocuHarnessX already ships `harnessx` as a direct git dependency — the - `ModelConfig.agentic` integration is proven and maintained. OpenUltraSAST reuses the - mechanism but tightens the posture (opt-in, not direct). -3. **Lower maintenance than vendoring.** Accept + SHA-pin gives reproducibility without a fork; - vendoring is paid only if the *installed* graph is itself judged unacceptable. - -### Preferred long-term: upstream dependency-extras (helps both projects) - -The clean fix for "I don't need most of these" is to modularize HarnessX's own dependencies into -extras — e.g. `harnessx[browser]`, `harnessx[sandbox-e2b]`, `harnessx[gateway]`, `harnessx[office]`, -`harnessx[otel]` — leaving a lean agent-runtime core (`anthropic`/`openai`/`litellm`/`pydantic`/ -`httpx`/`hydra`/`structlog`/`mcp`). Since the author owns the `Darwin-Agent/HarnessX` line, this is -actionable and benefits DocuHarnessX equally. OpenUltraSAST would then depend on -`harnessx` (lean core) under its `[harnessx]` extra and add only the subsets it needs. - -### Vendored lean subset (fallback, fully specified) - -If triggered, vendor `harnessx/` only: - -- **KEEP**: `core/`, `processors/{control,evaluation,observability,tools,context,memory,multi_model}`, - `meta_harness/`, `rl/`, `providers/`, `tools/builtin/` (minus `browser.py`/`web_*`), - `sandbox/{base,local,docker}.py`, plus `config`/`tracing`/`plugins`/`bundles`/`workspace` as needed. -- **REMOVE**: `frontend/`, `gateway/`, `harnessx/api/` (drops the web-server stack), `examples/`, - `docs/`, `benchmarks/`, `container/`, `scripts/`, `recipe/` (incl. the `recipe/verl_harnessX/verl` - submodule — clone non-recursive regardless), `extensions/skills/{docx,pdf,pptx,xlsx}` (drops - office/PDF), `harnessx/sandbox/e2b.py` (drops e2b), `harnessx/tools/builtin/browser.py` - (drops playwright + browser binary). Net ≈ 40–60% fewer packages, no browser binary. - -## Residual risks (security-tool-specific) and mitigations - -| Risk | Mitigation | -|---|---| -| `playwright install` fetches a Chromium binary out-of-band (unaudited, breaks hermetic builds) | Browser path is lazy and never invoked; document that `[harnessx]` does not trigger a browser download; lean subset deletes `browser.py` | -| `docker` SDK can reach the Docker daemon socket (privilege boundary) | Docker sandbox **default-off**, `LocalSandbox` default; run untrusted target code only inside the sandbox; SHA-pin docker/cryptography/cffi | -| FastAPI/uvicorn listening-socket surface in a security tool | `api/` server only starts on the explicit `serve` path; add a guard test asserting no server starts on import; lean subset removes `api/` | -| 118-package CVE surface in a supply-chain tool | SHA-pin the entire extra; keep core **zero-dep**; add a **zero-dep CI guard test** (importing `openultrasast` without the extra pulls nothing outside stdlib) | -| `recipe/verl` git submodule recursion | Init non-recursively; never in the accept-as-extra install path | - -> Decision date 2026-06-22. The `pip install --dry-run` audit is itself the Phase-3 gate: it is -> re-run before the packaging task (5.2), and if the realized graph pulls a browser binary / -> Docker SDK / FastAPI stack as *mandatory* (non-lazy) deps judged unacceptable, the spec switches -> to the vendored lean subset under the same extra name. diff --git a/.kiro/specs/harnessx-self-improving-rulesets/design.md b/.kiro/specs/harnessx-self-improving-rulesets/design.md deleted file mode 100644 index b6afacd..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/design.md +++ /dev/null @@ -1,614 +0,0 @@ -# Design Document - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Overview - -**Purpose.** This feature splits OpenUltraSAST's single overloaded regex layer — which today is detector, severity authority, and volume knob at once — into three governed owners: **rules detect**, a **central CWE policy decides severity and scope**, and a **single project score** becomes the optimization target. On top of that governance plane it adopts the HarnessX runtime exactly where it pays off — the LLM hunter loop, the LLM-judge verifier, and a meta-harness self-improvement loop that adapts rules and policy gates under hard bounds. - -**Users.** OpenUltraSAST maintainers gain a staged, reversible HarnessX adoption and an automatic, gated rule/policy tuning loop. Security policy owners gain a versioned, central CWE-to-severity policy. Security reviewers and CI consumers gain a bounded 0-100 AppSec project score with a two-condition gate. - -**Impact.** Today `findings.PatternRule` carries a string `severity`, `quick_scan_findings` reads a module-level `PATTERN_RULES`, `calibration.py` only demotes ranking (floored at `MIN_CALIBRATED_PRIORITY = 0.1`, keyed on `tags[0]`), `benchmark.py:368` writes an inert `next_improvement_candidate="rules_or_language_hunter"`, and `harness.HarnessRuntime.run_stage` is a sync dict-state driver. This design turns rules into TOML data, removes `PatternRule.severity`, sources severity from a vendored verycode `CWE_Score.tsv`, computes a project score, makes the benchmark actionable, and confines HarnessX behind an optional, SHA-pinned, lazy-imported extra — while `dependencies = []` and `requires-python = ">=3.11"` stay intact and the **≥90% recall / <10% FP** detection gate holds at every phase. - -### Goals - -- Rules become versioned data (`ruleset//*.toml`) with a loop-controlled `status` and no rule-local severity. -- One CWE = one governed severity, sourced from `policy/CWE_Score.tsv` via `resolve_severity()`. -- A single 0-100 `project_score` from `policy.severity × reachability`, with a two-condition CI gate. -- HarnessX adopted at the hunter loop, the verifier, and the meta-harness, behind `openultrasast[harnessx]`. -- A bounded self-improvement loop (`rule`, `policy` levers) that auto-reverts byte-for-byte under the gate. -- The governance + scoring + benchmark planes stay **zero-dependency** and **byte-identical by default** through Phases 0-2. - -### Non-Goals - -- The loop authoring or rewriting detection pattern text (human PR only). -- The loop editing the authoritative 0-5 CWE severity (verycode-upstream authority). -- A full single-step HarnessX rewrite of all deterministic stages and weight-level RL (deferred to optional Phase 5+). -- Changing the reachability vocabulary, which stays exactly `reachable | inferred-file-surface | unknown`. -- Dynamic/DAST scoring — report-only for a static tool. - -## Boundary Commitments - -### This Spec Owns - -- The **RulesetStore** (`ruleset/store.py` + `ruleset//*.toml`): `load_ruleset()`, the `PatternRule` schema (detection-only), and the `enabled|shadow|disabled` status vocabulary. -- The **PolicyStore** (`policy/verycode.py` + `policy/CWE_Score.tsv`): `load_policy()`, `CwePolicy`, `resolve_severity()`, and the CWE-120→CWE-121 remap. -- The **ScoreModel** (`scoring/project_score.py`): `SEV_WEIGHT`, `REACH_MULT`, `finding_penalty`, `project_score`, the score artifact, and the two-condition CI gate. -- The **loop ledger** (`.openultrasast/calibration/rule_policy.json`): the loop-owned overlay of `status`/evidence-floor/threshold/precision. -- The **HarnessX seam**: `harness/build_sast.py`, `HxScanOrchestrator`, the `MultiHookProcessor` SAST processors, the `SlotContractMixin`, and the `improve/` meta-harness driver. -- The **benchmark→signal** conversion: `rule_id` attribution, per-rule precision/recall, `rule_policy_delta`, and `rule_signals.json`. - -### Out of Boundary - -- The detection-pattern **content** of rules (authored by humans via PR; the loop may only toggle status / raise floors). -- The authoritative **0-5 severity** numbers in `CWE_Score.tsv` (verycode-upstream; PR gaps there). -- Upstream **HarnessX internals** beyond the minimal meta-harness patch (`_VALID_LEVERS`, `_render_task_brief`, `EvolveValidator`, changeset buckets, `contract_autocheck`). -- The DAST/dynamic execution engine; dynamic-only CWEs are report-only here. -- Retiring `HarnessRuntime` and adopting `rl/` — explicitly Phase 5+, optional. - -### Allowed Dependencies - -- **Stdlib only** for the governance/scoring/benchmark planes: `tomllib`, `csv`, `json`, `math`, `dataclasses`. -- **HarnessX** only behind the `[harnessx]` extra, pinned to a commit SHA, imported lazily behind `_has_harnessx()`. -- The existing `HarnessRuntime` as the top-level sync driver through Phases 0-4 (not removed by this spec until Phase 5). -- The existing reachability vocabulary emitted by `findings.py` (consumed, never redefined). - -### Revalidation Triggers - -- **Contract shape changes** to `PatternRule` (removing `severity`), `CwePolicy`, `VerificationResult`, `BenchmarkMiss`/`BenchmarkFalsePositive` (adding `rule_id`), or the `project_score` artifact JSON. -- **Data ownership change**: severity moving from rule → policy; calibration re-keying from `tags[0]` → `rule_id`. -- **Dependency direction change**: any new import of HarnessX outside a lazy `_has_harnessx()` guard; any addition to core `dependencies`. -- **Runtime prerequisite change**: the startup CWE-resolution invariant (fail-loud if an enabled rule's CWE is unmapped); the `pip install --dry-run` audit gate before Phase 3. -- **Lever surface change**: any addition to `journal._VALID_LEVERS` or the `EvolveValidator` bounds. - -## Architecture - -### Existing Architecture Analysis - -| Existing element | File | Constraint to respect / debt to fix | -|---|---|---| -| `HarnessRuntime.run_stage(name, callback)` | `harness.py:74` | Sync, dict `state` (`{"degradations": []}`), single `HarnessEvent` (`scan_id`), `ProcessorSpec.reads/writes` allow-list. Incompatible with HarnessX async-generator + message-window contract — **bridge, do not map**. Stays as the top-level driver through Phase 4. | -| `PatternRule.severity: str` + module-level `PATTERN_RULES` | `findings.py:31,34` | The fused defect: regex carries its own severity string ("high"/"medium"/"low") and is the only volume control. **Remove `severity`; externalize patterns to TOML.** | -| `quick_scan_findings(root, targets, rankings)` | `findings.py` / `cli.py:126` | Reads `PATTERN_RULES` implicitly. Becomes `quick_scan_findings(..., ruleset)` wired like `static_hints`. | -| `calibrate_rankings` demotion | `calibration.py:146` | `max(MIN_CALIBRATED_PRIORITY, calibrated - 0.5)`, keyed on `finding.tags[0]` (`calibration.py:262`). Demotes ranking, never the rule. **Re-key to `rule_id`; write status flips to ledger.** | -| `verify_findings` rejects on `reachability_status == "unknown"` | `verification.py:91` | Structural verifier. Keep as **zero-dep fallback**; add LLM-judge path. | -| `benchmark.py:368` `next_improvement_candidate="rules_or_language_hunter"` | `benchmark.py` | Inert stub written to `calibration_records.json`. **Make actionable** via `rule_id` attribution + `rule_policy_delta`. | -| `cli.py` `run_stage(...)` call chain | `cli.py:104-145` | The 13-stage sync pipeline. Stages migrate to HarnessX hooks incrementally; the hunter pool first. | - -### Architecture Pattern & Boundary Map - -**Selected pattern.** Three planes with one-directional authority: **governance data drives execution; execution emits signals; the improvement loop edits governance under a gate.** Nothing in execution mutates governance directly. The governance + scoring planes are **pure stdlib, zero HarnessX** (non-negotiable, so CI gating runs in the default install). HarnessX lives only in the execution and improvement planes, behind the extra. - -**Boundaries.** RulesetStore detects, PolicyStore decides severity/scope, ScoreModel optimizes — three owners, three change channels (human PR for patterns/severity; loop-via-`EvolveValidator` for status/floors/score-constants). - -```mermaid -flowchart TB - subgraph GOV["GOVERNANCE PLANE — stdlib, versioned, audited, ZERO-DEP"] - direction LR - POL["PolicyStore
policy/verycode.py
CWE_Score.tsv -> CwePolicy
severity 0-5, static/dynamic"] - RUL["RulesetStore
ruleset/store.py
*.toml: pattern, cwe, status"] - LED["rule_policy.json
loop-owned: status/floor/precision"] - SCO["ScoreModel
scoring/project_score.py
policy-weighted -> 0-100 + gate"] - FPL["FP ledger
false_positive_learnings.json"] - end - subgraph EXE["EXECUTION PLANE — HarnessX runtime, OPTIONAL EXTRA"] - direction LR - DET["Deterministic stages
skip_model MultiHookProcessors"] - HUNT["hunter_pool
HxScanOrchestrator (real LLM loop)"] - VER["VerifyProcessor
llm_judge helpers (+ structural fallback)"] - PSP["PolicyScoringProcessor + ScoreProcessor"] - end - subgraph IMP["IMPROVEMENT PLANE — meta_harness, OPTIONAL EXTRA"] - EVO["improve/evolve.py -> MetaAgent.evolve()
levers: rule | policy"] - BEN["BenchmarkHarness
recall/FP + per-rule attribution"] - end - POL --> DET & PSP & SCO - RUL --> DET & HUNT - LED --> RUL - DET --> HUNT --> VER --> PSP --> SCO - VER --> FPL - EXE -->|trajectories + verdicts| BEN - BEN -->|rule_signals.json + deltas| EVO - EVO -->|gated edits| RUL & POL & LED - SCO -->|gate verdict| CI[("CI gate")] - BEN -->|>=90% recall / <10% FP| CI -``` - -**New components rationale.** PolicyStore/ScoreModel/RulesetStore exist to break the fused regex defect; `HxScanOrchestrator`/`VerifyProcessor` exist to host the agent loop and judge; `improve/evolve.py`+`EvolveValidator` exist to make adaptation safe (bounded, gated, reversible). **Steering compliance:** zero-dep default install preserved; detection gate kept as a hard constraint separate from the score reward. - -### Technology Stack - -| Layer | Choice / Version | Role in Feature | Notes | -|-------|------------------|-----------------|-------| -| CLI | `cli.py` (`_run_scan`) | Wires ruleset/policy; Phase 3 routes hunter pool to `HxScanOrchestrator` | Unchanged entrypoints when extra absent | -| Governance | Python 3.11 stdlib (`tomllib`, `csv`, `dataclasses`) | RulesetStore + PolicyStore load | Zero-dep, fail-loud on unmapped CWE | -| Scoring | Python stdlib `math` | `project_score` exponential decay | Always on, zero-dep | -| Execution runtime | HarnessX `@ git+...@` (optional extra) | Hunter loop, judge verifier, processors | Lazy import behind `_has_harnessx()` | -| Self-improvement | HarnessX `meta_harness` (extra/vendored) | `MetaAgent.evolve` + `EvolveValidator` + journal | 5/6-hook minimal patch | -| Data / Storage | TOML rulesets, TSV policy, JSON ledger/artifacts | Versioned governance + loop overlay | All `_target_`/JSON-serializable for resume | -| Events | HarnessX lifecycle hooks (`on_task_start`…`on_task_end`) | Stage→hook mapping | `HarnessRuntime` sync driver retained Phases 0-4 | -| Packaging | `pyproject.toml` `[project.optional-dependencies]` | `harnessx` extra, pinned SHA | Core `dependencies = []` unchanged | - -## File Structure Plan - -### Directory Structure - -``` -src/openultrasast/ -├── ruleset/ -│ ├── store.py # load_ruleset()/write_ruleset(); PatternRule (NO severity); status/min_evidence_level/precision_estimate/version -│ ├── python/*.toml # rules-as-data, one [[rule]] per detector -│ ├── c/*.toml # incl. CWE-120 -> CWE-121 remapped buffer rules -│ └── /*.toml # same pattern per language -├── policy/ -│ ├── verycode.py # load_policy() -> dict[str, CwePolicy]; resolve_severity(); CwePolicy dataclass -│ └── CWE_Score.tsv # vendored verycode data (167 CWEs; header "Flaw Severity " has trailing space) -├── scoring/ -│ └── project_score.py # SEV_WEIGHT, REACH_MULT, finding_penalty(), project_score(), gate() -├── harness/ # Phase 3 (extra) -│ ├── build_sast.py # build_sast() factory mirrors examples/coding/build_coding -│ ├── task.py # ScanTask(BaseTask) -│ ├── run.py # HxScanOrchestrator: asyncio.run around model_config.agentic(cfg).run(...) -│ ├── slot_contract.py # SlotContractMixin: reads_slots/writes_slots on MultiHookProcessor -│ └── processors.py # SarifIngest/Preprocess/EntryPoint/Rank/Calibrate/QuickFindings/PolicyScoring/Verify/Score processors -└── improve/ # Phase 4 (extra) - ├── evolve.py # orchestrator: benchmark -> delta -> rule_signals.json -> MetaAgent.evolve() -> re-benchmark -> accept/revert - ├── rule_validator.py # EvolveValidator._rules / ._policy bound checks - └── rule_autocheck.py # ContractAutoCheckProcessor _check_rule/_check_policy branches - -.openultrasast/calibration/ -└── rule_policy.json # loop-owned ledger: status/threshold/evidence-floor/precision per rule_id -``` - -### Modified Files - -- `findings.py` — extract `PATTERN_RULES` to TOML; **delete `PatternRule.severity`**; `quick_scan_findings(root, targets, rankings, ruleset)` takes ruleset arg. -- `calibration.py` — re-key from `tags[0]` (line 262) to `rule_id`; write `rule_policy.json` status flips, not only ranking demotion (line 146). -- `benchmark.py` — add `rule_id` to `BenchmarkMiss`/`BenchmarkFalsePositive`; per-rule precision/recall in `BenchmarkMetrics`; emit `rule_policy_delta`; replace the inert stub at line 368; add `BenchEvaluator` subclass. -- `verification.py` — add LLM-judge path (`VerifyProcessor` over `llm_judge` helpers); keep structural verifier (`reachability_status == "unknown"` reject, line 91) as zero-dep fallback. -- `config.py` — ruleset/policy/score paths + gate thresholds (`K`, `MIN_SCORE`, `min_emit_priority`, `min_emit_precision`). -- `cli.py` — `_run_scan` wires ruleset/policy; Phase 3 routes the `hunter_pool` `run_stage` call (line 122) to `HxScanOrchestrator`. -- `harness.py` — retired in Phase 5 only. -- `pyproject.toml` — add `[project.optional-dependencies] harnessx = ["harnessx @ git+https://github.com/Darwin-Agent/HarnessX.git@"]`; core `dependencies = []` unchanged. -- `harnessx/meta_harness/{journal.py, agent.py, validate_workflow.py, processors/contract_autocheck.py}` — vendored/patched seam (Phase 4). - -## System Flows - -### Scan pipeline on HarnessX (Phase 3 seam) - -The top-level driver stays sync (`HarnessRuntime`); only the `hunter_pool` stage opens an async surface via `asyncio.run`. Deterministic stages run as `skip_model=True` processors carrying work in typed slots; the `SlotContractMixin` validates `reads_slots`/`writes_slots` on every hook (preserving the `ProcessorSpec.reads/writes` discipline). If a HarnessX capability is unavailable at runtime, the stage falls back to its existing equivalent and records a degradation in the manifest. - -```mermaid -flowchart TD - A["on_task_start: SarifIngestProcessor
-> slot sast.static_hints"] --> B["Preprocess -> sast.targets"] - B --> C["EntryPoint -> sast.entry_points"] - C --> D["on_step_start: Rank -> sast.rankings"] - D --> E["Calibrate (MemoryRetrieval pattern over FP ledger)
demotes sast.rankings"] - E --> F{"HarnessX extra
present & capable?"} - F -->|yes| G["hunter_pool: HxScanOrchestrator
asyncio.run(model_config.agentic(cfg).run(ScanTask))
control/tools/obs processors -> StatefulTrajectory"] - F -->|no / capability missing| G2["fallback run_hunter_pool()
record degradation in manifest"] - G --> H["QuickFindings: regex over load_ruleset()
-> sast.findings (NO severity yet)"] - G2 --> H - H --> I["PolicyScoringProcessor: resolve_severity(policy, rule.cwe)
+ emission decision; drop shadow findings"] - I --> J{"reachability_status
== unknown?"} - J -->|yes| K["VerifyProcessor: REJECTED (structural fallback)
or llm_judge verdict"] - J -->|no| K2["VerifyProcessor: llm_judge build/parse/call
(structural fallback if extra absent)"] - K --> L["record_calibration: lesson/missing -> FalsePositiveLearning
+ per-rule outcome -> rule_policy.json"] - K2 --> L - L --> M["on_task_end: ScoreProcessor -> project_score + gate"] - M --> N["report / sarif / manifest (incl. score block)"] -``` - -### Self-improvement loop - -The benchmark is the **regression suite governing the data stores**. The objective is `maximize project_score subject to recall ≥ 90% AND FP < 10%` as a hard feasibility constraint — kept separate so the loop cannot "improve" the number by shadowing rules that hurt real security. Every change is `shadow`-staged first, novelty/evidence/replay-gated, and reverted byte-for-byte on any failure. - -```mermaid -sequenceDiagram - participant B as BenchmarkHarness - participant O as improve/evolve.py - participant M as MetaAgent.evolve() - participant V as EvolveValidator (_rules/_policy) - participant J as journal (compute_attribution/fill_gating) - participant L as rule_policy.json - B->>O: misses(recall) + FPs/verifier-rejects(precision) + per-rule metrics - O->>O: write rule_signals.json [{rule_id, signal, cwe, evidence, suggested_action}] - O->>M: evolve(brief + rule_signals.json) - M->>M: write ruleset/*.toml + policy/overrides.toml + candidates.md + journal Round N - M->>V: validity(canon -> contract -> _rules/_policy -> replay) + policy(novelty + evidence) - alt validation fails - V-->>M: StrictValidationError(kind="rule_change") -> round rejected - else passes - V->>B: re-run benchmark on candidate - B->>J: passed_now vs passed_before - J->>J: compute_attribution(predicted): flipped/still_F/regressed/still_T/absent - alt recall<90% OR FP>=10% OR score regressed OR unpredicted regression - J->>L: fill_gating(reverted) -> restore config byte-for-byte - else - J->>L: fill_gating(accepted) -> persist status/floor/precision - end - end -``` - -## Requirements Traceability - -| Requirement | Summary | Components | Interfaces | Flows | -|-------------|---------|------------|------------|-------| -| 1.1, 1.4 | Hunter pool on real `Harness.run()`; async confined to one stage | `HxScanOrchestrator` | `run.py:HxScanOrchestrator.run()` | Scan pipeline | -| 1.2 | Verify via llm-judge + structural fallback | `VerifyProcessor` | `processors.py:VerifyProcessor` | Scan pipeline | -| 1.3 | Extra absent → unchanged quick/standard | `cli._run_scan`, `_has_harnessx()` | capability guard | Scan pipeline (`F` branch) | -| 1.5 | Slot allow-list validated per hook | `SlotContractMixin` | `slot_contract.py` | Scan pipeline | -| 1.6 | Deterministic stages `skip_model` | SAST processors | `MultiHookProcessor(skip_model=True)` | Scan pipeline | -| 1.7 | Event/`scan_id` rename to HarnessX hooks/`run_id` | `build_sast`, journal | hook map table | Scan pipeline | -| 1.8 | Capability-missing fallback + manifest degradation | `HxScanOrchestrator` | manifest `degradations` | Scan pipeline (`G2`) | -| 2.1, 2.4 | Rules-as-data; 3 statuses; no rule severity | RulesetStore | `load_ruleset()`, `PatternRule` | — | -| 2.2, 2.3 | Severity from policy; legacy strings discarded | PolicyStore, `PolicyScoringProcessor` | `resolve_severity()` | Scan pipeline (`I`) | -| 2.5 | Shadow excluded from report+score | `PolicyScoringProcessor`, ScoreModel | emission filter | Scan pipeline (`I`,`M`) | -| 2.6 | CwePolicy 0-5 + static/dynamic; dynamic report-only | PolicyStore | `CwePolicy` | — | -| 2.7 | Fail-loud on unmapped enabled-rule CWE | PolicyStore startup invariant | `load_policy()` check | — | -| 2.8 | CWE-120 → CWE-121 remap | RulesetStore | TOML data | — | -| 2.9, 5.9, 7.5 | Governance/score/benchmark zero-dep | all GOV components | stdlib-only | — | -| 3.1 | Two levers `rule`/`policy` | `improve/evolve.py` | `_VALID_LEVERS` | Self-improvement | -| 3.2, 3.3 | No pattern text / no 0-5 severity edits | `EvolveValidator` | `_rules`/`_policy` | Self-improvement | -| 3.4 | No enabled→disabled jump (shadow mandatory) | `EvolveValidator._rules` | bound check | Self-improvement | -| 3.5 | Parse/status/bounds/dup/CWE-resolve checks | `EvolveValidator._rules` | validity phase | Self-improvement | -| 3.6 | Novelty gate on reverted re-proposals | `journal.check_novelty` | novelty gate | Self-improvement | -| 3.7 | Replay smoke gate (ruleset boots) | `run_synthetic_task_smoke_gate` | replay phase | Self-improvement | -| 3.8, 3.9 | `StrictValidationError`; journal each edit | `EvolveValidator`, journal | `fill_gating` | Self-improvement | -| 4.1, 4.2 | `rule_id` attribution; per-rule metrics; delta artifact | BenchmarkHarness | `rule_policy_delta` | Self-improvement | -| 4.3 | Delta → `rule_signals.json` (miss/fp) | `improve/evolve.py` | signal writer | Self-improvement | -| 4.4, 4.5 | Hard accept gate; byte-for-byte revert | `improve/evolve.py`, journal | `fill_gating(reverted)` | Self-improvement | -| 4.6 | Auto-shadow precision-draggers | `improve/evolve.py` | status flip | Self-improvement | -| 4.7 | Coverage-gap nominates rule (human PR) | `improve/evolve.py` | `rule_signals.json` | Self-improvement | -| 4.8, 6.5 | Score=reward, gate=feasibility (separate) | `improve/evolve.py` | objective split | Self-improvement | -| 5.1, 5.2 | 0-100 score; reachability multipliers | ScoreModel | `project_score()` | Scan pipeline (`M`) | -| 5.3 | Dynamic-only/unmapped → 0 penalty, report-only | ScoreModel | `finding_penalty()` | Scan pipeline (`M`) | -| 5.4 | Shadow excluded from score input | ScoreModel | pre-call filter | Scan pipeline (`M`) | -| 5.5 | Score artifact + manifest block | ScoreModel | score JSON | Scan pipeline (`N`) | -| 5.6 | FP lowers effective reachability, not delete | `calibration.py` | reachability knob | Self-improvement | -| 5.7 | Hard gate: sev5 + reachable → fail | CI Score Gate | `gate()` | — | -| 5.8 | Score gate advisory-first, then blocking | CI Score Gate | `gate()` | — | -| 6.1, 6.4, 6.6 | Per-merge recall/FP gate + HX tolerance | CI Pipeline | benchmark gate | — | -| 6.2 | Byte-identical baseline Phases 0-2 | BenchmarkHarness | golden diff | Migration | -| 6.3 | Gate must pass on re-benchmarked candidate | `improve/evolve.py` | accept gate | Self-improvement | -| 7.1, 7.6 | Empty core deps; pinned extra; non-recursive submodule | Packaging Config | `pyproject.toml` | Migration | -| 7.2 | Lazy import behind capability check | `_has_harnessx()` | guard | — | -| 7.3, 7.4, 7.7 | Dry-run audit; vendored fallback trigger; pre-specified | Packaging Config | audit gate | Migration (Phase 3 gate) | - -## Components and Interfaces - -| Component | Domain/Layer | Intent | Req Coverage | Key Dependencies (P0/P1) | Contracts | -|-----------|--------------|--------|--------------|--------------------------|-----------| -| RulesetStore | Governance | Rules-as-data + status | 2.1,2.4,2.8 | tomllib (P0), rule_policy.json (P1) | State | -| PolicyStore | Governance | CWE→severity/scope authority | 2.2,2.6,2.7 | CWE_Score.tsv (P0) | State | -| ScoreModel | Scoring | 0-100 score + gate | 5.1-5.9 | PolicyStore (P0) | Batch, State | -| HxScanOrchestrator | Execution | Hunter loop on HarnessX | 1.1,1.4,1.8 | HarnessX (P0), RulesetStore (P1) | Service, Event | -| PolicyScoringProcessor | Execution | Resolve severity + emission | 2.2,2.3,2.5 | PolicyStore (P0) | Event | -| VerifyProcessor | Execution | LLM-judge + structural fallback | 1.2 | llm_judge (P1) | Event | -| EvolveValidator (`_rules`/`_policy`) | Improvement | Bound-check loop edits | 3.2-3.8 | journal (P0), PolicyStore (P1) | Service | -| improve/evolve.py | Improvement | Benchmark→signal→evolve→gate | 4.1-4.8,6.3 | BenchmarkHarness (P0), MetaAgent (P0) | Batch | - -### Governance Layer - -#### RulesetStore - -| Field | Detail | -|-------|--------| -| Intent | Load detection rules from versioned TOML; carry loop-controlled status; carry no severity | -| Requirements | 2.1, 2.4, 2.8, 2.9 | - -**Responsibilities & Constraints** -- `load_ruleset(path) -> tuple[PatternRule, ...]` replaces module-level `PATTERN_RULES`; loaded with `tomllib` (zero-dep). -- `PatternRule` carries `rule_id`, `languages`, `cwe`, `tags`, `pattern`, `status`, `min_evidence_level`, `precision_estimate`, `version` — **no `severity` field**. -- `status ∈ {enabled, shadow, disabled}` is the volume control; the loop overlays `status`/`min_evidence_level`/`precision_estimate` from `rule_policy.json`. -- C/C++ buffer rules (`c-unsafe-copy`, `c-unsafe-memory-copy`, `c-unsafe-scanf`) declare `cwe = "CWE-121"` (remapped from unmapped CWE-120). - -**Dependencies** — Inbound: `quick_scan_findings` (P0). Outbound: `rule_policy.json` overlay (P1). External: `tomllib` (P0). - -**Contracts**: State [x] - -##### Service Interface -```python -def load_ruleset(path: Path, ledger: Path | None = None) -> tuple[PatternRule, ...]: ... -def write_ruleset(path: Path, rules: Iterable[PatternRule]) -> None: ... - -@dataclass(frozen=True) -class PatternRule: - rule_id: str - languages: tuple[str, ...] - cwe: str # the ONLY policy join key - tags: tuple[str, ...] - pattern: str # human-PR-only; loop must never edit - status: str = "enabled" # enabled | shadow | disabled (loop-controlled) - min_evidence_level: str = "static_corroboration" - precision_estimate: float = 0.0 - version: str = "1" -``` -- **Preconditions:** every loaded rule has a non-empty `pattern` and a `CWE-NNN` `cwe`. -- **Postconditions:** ledger overlay applied; `status`/floors reflect `rule_policy.json`. -- **Invariants:** no `severity` attribute exists; `status` is one of the three literals. - -#### PolicyStore - -| Field | Detail | -|-------|--------| -| Intent | Authoritative CWE→severity/scope; resolve finding severity | -| Requirements | 2.2, 2.3, 2.6, 2.7, 2.8 | - -**Responsibilities & Constraints** -- `load_policy() -> dict[str, CwePolicy]` keyed `"CWE-NNN"` from `CWE_Score.tsv` (167 CWEs; severity column header is literally `"Flaw Severity "` with a trailing space — parse defensively). -- `resolve_severity(policy, rule.cwe, target, ranking)` replaces every literal `rule.severity` read. **Policy always wins**; any legacy rule severity string is discarded. -- **Startup invariant (fail-loud):** every `enabled` rule's `cwe` must resolve in PolicyStore, else the scan aborts. `dynamic`-only CWEs are report-only. - -**Dependencies** — Inbound: `PolicyScoringProcessor`, ScoreModel (P0). External: `csv`, `dataclasses` (P0). - -**Contracts**: State [x] - -##### Service Interface -```python -@dataclass(frozen=True) -class CwePolicy: - flaw_category: str - severity: int # 0..5 — AUTHORITATIVE; overrides any rule string; loop must never edit - static: bool # SAST scope gate - dynamic: bool # DAST scope — report out-of-scope, never scored by static tool - -def load_policy(tsv: Path = CWE_SCORE_TSV) -> dict[str, CwePolicy]: ... -def resolve_severity(policy: Mapping[str, CwePolicy], cwe: str, target, ranking) -> int: ... -def assert_rules_resolve(rules, policy) -> None: # raises at startup if an enabled rule's CWE is unmapped - ... -``` -- **Invariants:** one CWE ⇒ one severity; two rules for the same CWE always receive identical severity. - -### Scoring Layer - -#### ScoreModel - -| Field | Detail | -|-------|--------| -| Intent | Compute a 0-100 project score and the two-condition gate verdict | -| Requirements | 5.1-5.9 | - -**Responsibilities & Constraints** -- Pure stdlib (`math`), always on, zero-dep — runs in the default install with no HarnessX. -- Excludes `shadow`-status findings before computing; assigns 0 penalty to dynamic-only/unmapped CWEs (reports them instead). -- The `REACH_MULT` keys are exactly the three values `findings.py` emits; the reachability multiplier is the **FP-calibration knob** (a confirmed FP lowers effective reachability, never deletes a rule). -- Emits the scan-level artifact and the manifest score block; computes the hard gate (sev5 + reachable) and the score gate (advisory-first). - -**Contracts**: Batch [x] / State [x] - -##### Service Interface -```python -SEV_WEIGHT = {5: 50, 4: 25, 3: 10, 2: 2, 1: 1, 0: 0} -REACH_MULT = {"reachable": 1.0, "inferred-file-surface": 0.6, "unknown": 0.4} - -def finding_penalty(f, rule_cwe: str, policy) -> float: - pol = policy.get(rule_cwe) - if pol is None or not pol.static: # dynamic-only / unmapped -> report, don't score - return 0.0 - return SEV_WEIGHT[pol.severity] * REACH_MULT.get(f.reachability_status, 0.4) - -def project_score(findings, rule_cwe_by_id, policy, K: float = 60.0) -> int: - # shadow-status findings excluded before this call - P = sum(finding_penalty(f, rule_cwe_by_id[f.finding_id], policy) for f in findings) - return round(100 * math.exp(-P / K)) # P=0 -> 100; one critical-reachable -> ~43 - -def gate(findings, score, *, min_score: int, block_severity_reachable: int = 5, - blocking: bool) -> dict: ... -``` -- **Postconditions:** artifact JSON (see Data Models) written next to per-finding output and merged into the manifest. - -### Execution Layer - -#### HxScanOrchestrator - -| Field | Detail | -|-------|--------| -| Intent | Run the hunter pool on a real HarnessX agent loop, confining async to one stage | -| Requirements | 1.1, 1.4, 1.7, 1.8 | - -**Responsibilities & Constraints** -- Invoked from the existing `hunter_pool` `run_stage` callback (`cli.py:122`); wraps `asyncio.run(model_config.agentic(harness_config).run(ScanTask(...)))`. -- `build_sast()` mirrors `examples/coding/build_coding`; `ModelConfig(main=hunter, judge=…, ranker=…, patcher=…).agentic(cfg)` keeps model info out of `HarnessConfig`. -- Each hunter sub-harness registers `LoopDetectionProcessor`, `CostGuardProcessor`/`TokenBudgetProcessor`, `ToolFailureGuard`, `ToolFilterProcessor`, `SelfVerifyProcessor`, `EpisodeMetricsProcessor`/`OTelProcessor`/`CheckpointProcessor`. -- Produces a `StatefulTrajectory` per target; rewards backfilled from verifier verdicts. On capability-missing, falls back to `run_hunter_pool()` and appends to manifest `degradations`. - -**Dependencies** — External: HarnessX core + processors (P0, lazy). Inbound: `cli._run_scan` (P0). - -**Contracts**: Service [x] / Event [x] - -##### Event Contract -- Hook mapping: `stage_start`→`on_step_start`, `stage_end`→`on_step_end`, `scan_id`→`session_id`/`run_id`; `before/after_model|tool` direct. `HARNESSX_CONTRACT_MODE` (message-window) + `OUS_SLOT_CONTRACT` (key-level via `SlotContractMixin`) split the old single `contract_mode`. - -#### PolicyScoringProcessor - -| Field | Detail | -|-------|--------| -| Intent | Resolve policy severity per finding and make the emission decision | -| Requirements | 2.2, 2.3, 2.5 | - -**Responsibilities & Constraints** -- Runs `on_after_tool`/`on_step_end`; reads `sast.rankings` + PolicyStore; writes resolved severity + emit-decision. -- Drops `shadow` findings from the reported set and the score input while recording their outcome for precision tracking. -- Applies `min_emit_priority`/`min_emit_precision` thresholds — the new volume controls that replace "every regex match becomes a finding". - -**Contracts**: Event [x] - -#### VerifyProcessor - -| Field | Detail | -|-------|--------| -| Intent | LLM-judge verification with a zero-dep structural fallback | -| Requirements | 1.2 | - -**Responsibilities & Constraints** -- Wraps `build_judge_prompt`/`parse_judge_response`/`_call_judge` (`llm_judge.py`) as a `judge`-role sub-harness; maps the strict-JSON verdict (`plausible|unsupported|hedging|…` + `lesson`/`missing`) to `VerificationResult`. -- **Fallback:** the existing structural verifier (`verification.py:91`, rejects on `reachability_status == "unknown"`) is used verbatim when the extra is absent or the judge is unavailable. - -**Contracts**: Event [x] - -### Improvement Layer - -#### EvolveValidator (`_rules` / `_policy`) - -| Field | Detail | -|-------|--------| -| Intent | Enforce the bounded-lever safety contract on every loop edit | -| Requirements | 3.2-3.8 | - -**Responsibilities & Constraints** (the gate — runs in the *validity* phase beside `_contract`/`_replay`) -- `_rules`: edit parses; `status ∈ {enabled,shadow,disabled}`; `tighten`/`loosen` adjusts only threshold/evidence-floor **within configured bounds**; `add` doesn't duplicate a `rule_id`; **every `cwe` resolves in PolicyStore**; **pattern text unchanged** (reject otherwise); **no `enabled→disabled` jump** (require `shadow`). -- `_policy`: touches only evidence floor, scope, `K`/`MIN_SCORE` within bounds; **never the 0-5 severity**. -- On any failure: `raise StrictValidationError(kind="rule_change")` → the whole round is rejected. - -**Contracts**: Service [x] - -##### Service Interface -```python -_VALID_LEVERS = frozenset({ - "configuration", "control", "action", "instruction", - "rule", # status/evidence-floor/threshold in ruleset/ — NEVER pattern text - "policy", # evidence floor + scope + K/MIN_SCORE — NEVER the 0-5 CWE severity -}) - -class EvolveValidator: - def _rules(self, edit) -> None: ... # raises StrictValidationError(kind="rule_change") - def _policy(self, edit) -> None: ... -``` - -#### improve/evolve.py (orchestrator) - -| Field | Detail | -|-------|--------| -| Intent | Drive benchmark→signal→evolve→re-benchmark→accept/revert under the gate | -| Requirements | 4.1-4.8, 6.3 | - -**Responsibilities & Constraints** -- Converts `rule_policy_delta` + FP learnings → `rule_signals.json` (recall side `signal="miss"`; precision side `signal="verifier_reject"|"fp"`); calls `MetaAgent.evolve()`. -- **Acceptance gate (hard):** round accepted **only if** aggregate recall ≥ 90% **AND** FP < 10% **AND** `project_score` did not regress on the held set **AND** no unpredicted regression; otherwise `fill_gating(outcome="reverted")` restores config byte-for-byte. -- Precision-dragging rules are **auto-shadowed**, never deleted; coverage gaps nominate a rule via `rule_signals.json` (pattern authoring stays human-PR). -- Objective: `maximize project_score s.t. recall/FP gate` — score is the reward, the gate is the feasibility region; never folded together. - -**Contracts**: Batch [x] - -## Data Models - -### Ruleset policy schema (detection only — TOML) - -```toml -[[rule]] -rule_id = "python-sql-injection" -languages = ["python"] -cwe = "CWE-89" # the ONLY policy join key -tags = ["injection"] -pattern = '(\.execute)\s*\(\s*[^,)]*%' # human-PR-only -status = "enabled" # enabled | shadow | disabled (loop-controlled) -min_evidence_level = "static_corroboration" # loop may raise within bounds -precision_estimate = 0.62 # backfilled from benchmarks -version = "3" -``` - -Status semantics: `enabled` fires/scores/reports; `shadow` fires and records outcomes but is excluded from report and score (mandatory staging state for any loop-proposed change); `disabled` does not fire. - -Loop ledger (`rule_policy.json`) — loop-owned overlay, re-keyed on `rule_id`: -```json -{ "python-sql-injection": { "status": "shadow", "min_evidence_level": "dynamic_corroboration", - "precision_estimate": 0.41, "outcome": "accepted", "round": 7 } } -``` - -### Project-score schema (artifact + manifest block) - -```json -{ "project_score": 73, "max_severity": 5, "penalty_total": 30.0, - "by_category": {"Command or Argument Injection": 1, "Cryptographic Issues": 2}, - "out_of_scope_dynamic_only": 0, - "unmapped_cwe": [], - "gate": {"min_score": 80, "block_severity_reachable": 5, "blocking": false, "passed": false} } -``` - -### Consistency & Integrity - -- **Authority direction:** governance data is the source of truth; execution reads it, emits findings/signals; only the loop (via `EvolveValidator`) writes `status`/floors/score-constants back, never the pattern or the 0-5 severity. -- **Temporal/versioning:** `PatternRule.version` and `rule_policy.json.round` give per-rule audit; the journal records each lever edit; `fill_gating(reverted)` restores byte-for-byte. -- **Serialization (resume-safe):** all scan slots (`sast.targets`, `sast.findings`, `sast.rankings`, verifications) use plain dicts/dataclasses with `_target_` registration so they survive `wake()`. - -## Error Handling - -### Error Strategy - -| Error type | Condition | Response | -|---|---|---| -| **Startup config error** | An `enabled` rule's `cwe` is unmapped in PolicyStore | `assert_rules_resolve` raises; **fail loud**, abort scan before any work (Req 2.7) | -| **Malformed ruleset/policy TOML** | `tomllib` parse failure | Fail loud at load; CI catches before merge | -| **Loop edit violates bounds** | Pattern edit / severity edit / enabled→disabled / duplicate `rule_id` / unmapped CWE | `EvolveValidator` raises `StrictValidationError(kind="rule_change")`; entire round rejected (Req 3.8) | -| **Reverted re-proposal w/o rationale** | Novelty gate hit | `journal.check_novelty` blocks; loop continues with other candidates (Req 3.6) | -| **Replay smoke gate failure** | Edited ruleset/policy won't boot | Round rejected before benchmark re-run (Req 3.7) | -| **Acceptance-gate failure** | recall<90% OR FP≥10% OR score regressed OR unpredicted regression | `fill_gating(reverted)` restores config byte-for-byte (Req 4.5) | -| **HarnessX capability missing at runtime** | Extra absent or sub-harness unavailable | Stage falls back to existing equivalent; record degradation in manifest (Req 1.8) | -| **Slot contract violation** | Stage reads/writes an undeclared slot | `SlotContractMixin` raises under `OUS_SLOT_CONTRACT=strict`; warns under `warn` (Req 1.5) | -| **Slot serialization loss on resume** | Non-`_target_` slot restores as `None` | Per-slot-type round-trip test guards; `_target_` registration required | - -### Monitoring - -- `EpisodeMetricsProcessor.episode_summary` → manifest metrics block; `OTelProcessor` OTLP export when configured. -- `HarnessJournal` writes per-run `{run_id}.jsonl` / `_trace.jsonl` / `_state.json`, rotated on `SegmentBoundaryEvent`. -- The score artifact `unmapped_cwe` list surfaces policy-completeness drift; the journal changeset table surfaces every rule/policy lever edit per round. - -## Testing Strategy - -### Unit Tests -- `load_policy()` parses the `"Flaw Severity "` trailing-space header and yields 167 `CwePolicy` entries with correct `static`/`dynamic` flags. -- `project_score()` boundary cases: P=0 → 100; one severity-5 reachable finding → ~43; one low finding → ~97; shadow findings excluded. -- `resolve_severity()` returns policy severity (not the legacy rule string) and is identical for two rules sharing a CWE. -- `EvolveValidator._rules` rejects: pattern edit, severity edit, `enabled→disabled` jump, duplicate `rule_id`, unmapped CWE. -- `finding_penalty()` returns 0.0 for dynamic-only and unmapped CWEs. - -### Integration Tests -- **Byte-identical baseline (Req 6.2):** with the default ruleset and an empty `rule_policy.json`, `benchmark.py` output matches the pre-feature golden corpus exactly. -- **Startup invariant (Req 2.7):** an enabled rule with an unmapped CWE aborts the scan with a loud error. -- **Self-improvement round (Req 4.4/4.5):** a deliberately noisy rule is auto-shadowed when it drags precision; a round that regresses recall is reverted byte-for-byte. -- **Capability fallback (Req 1.8):** extra absent → `quick`/`standard` run through the sync driver unchanged; degradation recorded when a HX stage is requested but unavailable. -- **Slot round-trip (Req on resume):** each scan slot type serializes and restores without loss after `wake()`. - -### E2E / Gate Tests -- **CI detection gate (Req 6.1):** the pipeline fails the merge if recall<90% or FP≥10% on the corpus. -- **Two-condition score gate (Req 5.7/5.8):** hard gate fails on a sev5+reachable finding; score gate is advisory until `K`/`MIN_SCORE` are calibrated, then blocking. -- **Zero-dep guard (Req 7.5):** with HarnessX absent, governance/scoring/benchmark planes operate and CI gating runs in the default install. - -### Performance / Tolerance -- **HX tolerance (Req 6.4):** an enabled HarnessX-backed stage keeps recall/FP within the configured tolerance of the zero-dep baseline. -- Hunter-loop cost/token budgets enforced by `CostGuardProcessor`/`TokenBudgetProcessor`. - -## Security Considerations - -- **Supply-chain posture (Req 7):** core `dependencies = []` is preserved; HarnessX ships only as a SHA-pinned optional extra, lazy-imported behind `_has_harnessx()`. A `pip install --dry-run` audit runs before Phase 3; if the realized transitive graph pulls browser binaries / Docker SDK / a FastAPI web stack as *mandatory* deps and maintainers judge it unacceptable for a supply-chain auditing tool, the pre-specified **vendored lean subset** (Option C) is switched in under the same extra name — a packaging change, not a redesign. Submodule init is **non-recursive** so the large `verl` RL fork is never pulled. -- **Loop containment:** the meta-agent can never invent or rewrite detectors (no pattern edits) or rewrite the severity authority (no 0-5 edits); `shadow`-first staging, novelty/evidence/replay gates, and byte-for-byte revert bound its blast radius — directly preventing the "too influential" defect from re-entering via the loop. -- **Gate integrity:** the recall/FP gate is a hard constraint that is never folded into the score reward, so the loop cannot "improve" the number by shadowing rules that hurt real security. - -## Migration Strategy - -Every phase is independently shippable, test-gated, and reversible. Phases 0-2 deliver governance value at **zero dependency cost** and stay byte-identical by default. Phases 3-4 are the HarnessX adoption behind the extra. Phase 5 is the optional native end-state. - -```mermaid -flowchart LR - P0["Phase 0
Policy + Score
zero-dep, gate advisory"] --> P1["Phase 1
Rules-as-data
zero-dep, byte-identical"] - P1 --> P2["Phase 2
Benchmark->signal + ledger
zero-dep"] - P2 --> GATE{"pip --dry-run audit
(D2 decision)"} - GATE -->|graph acceptable| P3B["Phase 3
[harnessx] extra (Option B)
hunter loop + verify"] - GATE -->|graph unacceptable| P3C["Phase 3
vendored lean subset (Option C)
same extra name"] - P3B --> P4["Phase 4
Self-improvement loop live"] - P3C --> P4 - P4 --> P5["Phase 5 (optional)
full single-step harness + RL
retire HarnessRuntime"] -``` - -| Phase | Scope | Dep cost | Merge gate | Reversibility | -|---|---|---|---|---| -| **0 — Policy & score** | Vendor `CWE_Score.tsv`; `policy/verycode.py` + `scoring/project_score.py`; CWE-120→121 remap; startup CWE-resolution invariant; emit score artifact + advisory gate. `HarnessRuntime`/`PATTERN_RULES` unchanged. | **zero** | benchmark byte-identical; score is additive output | gate disable flag | -| **1 — Rules as data** | Extract `PATTERN_RULES`→`ruleset//*.toml`; delete `PatternRule.severity`; `load_ruleset()`; `resolve_severity`; add `status` + emit thresholds. | **zero** | benchmark **byte-identical** to Phase 0 | keep `PATTERN_RULES` fallback | -| **2 — Benchmark→signal + ledger** | `rule_id` on misses/FPs; per-rule precision/recall; emit `rule_policy_delta`; `rule_policy.json` ledger; calibration writes status flips re-keyed on `rule_id`. | **zero** | empty ledger == current behavior | empty ledger = no-op | -| **3 — HX execution adoption** | Run `pip --dry-run` audit (D2 gate). Add `[harnessx]` extra (pinned SHA, lazy, zero-dep CI guard); `build_sast()` + `HxScanOrchestrator`; hunter pool on real `Harness.run()`; `VerifyProcessor`; `PolicyScoring`/`ScoreProcessor`. `HarnessRuntime` stays. | **opt-in heavy** | extra-absent: unchanged; extra-present: recall/FP within tolerance | feature flag off | -| **4 — Self-improvement live** | Vendor/patch the meta-harness seam; orchestrator benchmark→delta→signal→`evolve()`→re-benchmark→accept/revert. | **opt-in** | ≥90% recall / <10% FP per-round; auto-revert | don't run `evolve` → ledger-only changes | -| **5 — Full harness + RL (optional)** | Migrate deterministic stages to single-step harness + `SlotContractMixin`; retire `HarnessRuntime`; promote score gate to blocking; optional `rl/`. | **opt-in** | golden-diff vs Phase 3 empty; gate held | per-stage revert | - -**Rollback triggers.** Any phase that drops recall<90% or raises FP≥10% on the corpus blocks the merge; any self-improvement round failing the acceptance gate reverts byte-for-byte; an unacceptable `pip --dry-run` graph switches Phase 3 to the vendored subset. diff --git a/.kiro/specs/harnessx-self-improving-rulesets/requirements.md b/.kiro/specs/harnessx-self-improving-rulesets/requirements.md deleted file mode 100644 index 46cc369..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/requirements.md +++ /dev/null @@ -1,120 +0,0 @@ -# Requirements Document - -> **Status 2026-10-02:** retired with HarnessX (see the note below). Its task 6.3 modules `stage_processors.py` and `slot_contract.py` were deleted on 2026-10-02: nothing imported them. Kept as decision history; nothing in src, tests or CI points here. - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Introduction - -OpenUltraSAST today fuses three responsibilities into one regex layer: detection patterns also carry their own severity strings and act as the sole volume control, so a noisy rule can dominate scan output, set its own criticality, and fire forever with no central governance. This feature splits those responsibilities into three governed owners — rules detect, a central CWE policy decides severity and scope, and a single project score becomes the optimization target — and adopts the HarnessX runtime exactly where it pays off: the LLM hunter loop, the LLM-judge verifier, and a meta-harness self-improvement loop that adapts rules and policy gates under hard bounds. - -The work spans five user asks. First, OpenUltraSAST shall adopt HarnessX directly at the hunter loop and migrate the existing scan flow stage-by-stage onto HarnessX processors. Second, detection rules shall become versioned data governed centrally, with severity sourced from the verycode CWE policy rather than from the rule itself. Third, the HarnessX meta-harness shall adapt rules and policy gates within bounded levers. Fourth, benchmark results shall drive that adaptation as actionable signals with a hard acceptance gate. Fifth, a Veracode-style project score shall be computed from policy-weighted, reachability-adjusted findings and gated in CI. - -Two cross-cutting constraints govern every part of this feature. The existing detection gate of at least 90% recall and under 10% false-positive rate on the benchmark corpus must never regress, at any phase, for any change. And the HarnessX runtime must be adopted in a way that preserves the project's zero-dependency posture for the governance and scoring planes, so an explicit dependency-weight decision controls whether HarnessX ships as an optional extra or as a vendored lean subset. - -## Boundary Context - -- **In scope**: detection rules as versioned data; a central CWE-to-severity policy with static/dynamic scope; a single project score (0-100) with a two-condition CI gate; HarnessX adoption at the hunter loop, the verifier, and the self-improvement meta-harness; benchmark-to-signal conversion with per-rule attribution; bounded loop authority over rule status, evidence floors, thresholds, and score constants; a dependency-weight audit and decision; preservation of the recall/FP detection gate. -- **Out of scope**: authoring new detection pattern text by the loop (human pull-request only); editing the authoritative 0-5 CWE severity by the loop (verycode-upstream authority); a full single-step HarnessX rewrite of all deterministic stages and weight-level reinforcement learning (deferred, optional, later phase); changing the reachability vocabulary, which remains exactly `reachable | inferred-file-surface | unknown`; dynamic/DAST scoring, which is report-only for a static tool. -- **Adjacent expectations**: the governance and scoring planes shall run with no HarnessX present and `dependencies = []`; CI gating (score plus benchmark) shall run in the zero-dependency default install; HarnessX, when present, shall be confined behind a capability check and lazy imports so heavy code paths stay cold in the default install. - -## Requirements - -### Requirement 1: HarnessX Direct Adoption and Scan-Flow Migration - -**Objective:** As an OpenUltraSAST maintainer, I want the scan flow to adopt the HarnessX runtime directly at the LLM hunter loop and migrate the existing stages onto HarnessX processors in a staged way, so that the project gains the HarnessX agent loop, journal, and trajectory machinery without a high-risk big-bang rewrite. - -#### Acceptance Criteria -1. Where the HarnessX extra is installed, the Scan Orchestrator shall run the hunter pool stage on a real HarnessX `Harness.run()` loop with the HarnessX control, tool, and observability processors registered in the hunter sub-harness. -2. Where the HarnessX extra is installed, the Verify Processor shall produce verification verdicts using the HarnessX llm-judge helpers and shall retain the structural verifier as a zero-dependency fallback. -3. While the HarnessX extra is absent, the Scan Runtime shall execute `quick` and `standard` scans through the existing sync driver with behavior unchanged from the pre-adoption baseline. -4. When the hunter pool stage runs under HarnessX, the Scan Orchestrator shall confine all asynchronous execution to that single stage so that the deterministic stages remain synchronous. -5. The Scan Runtime shall preserve each stage's declared read and write slot allow-list and shall validate slot access on every lifecycle hook, so that the existing provenance and audit discipline is retained under the HarnessX runtime. -6. Where a deterministic stage runs under HarnessX, the Scan Runtime shall execute it with model invocation disabled so that no deterministic stage incurs a model call. -7. When migrating event names, the Scan Runtime shall map the existing `stage_start`/`stage_end` to the HarnessX step-level hooks and `scan_id` to the HarnessX run identifier without losing trace continuity. -8. If a HarnessX-backed stage is enabled and the corresponding HarnessX capability is unavailable at runtime, then the Scan Runtime shall fall back to the existing equivalent stage and shall record the degradation in the scan manifest. - -### Requirement 2: Central Security-Affine Ruleset Governance - -**Objective:** As a security policy owner, I want detection rules to be versioned data whose severity and scope come from a central CWE policy rather than from the rule itself, so that one CWE maps to one governed severity and no rule can be too influential by setting its own criticality or volume. - -#### Acceptance Criteria -1. The Ruleset Store shall load detection rules from versioned data files in which each rule declares a pattern, a CWE, tags, and a loop-controlled `status`, and shall not carry any rule-local severity field. -2. When resolving the severity of a finding, the Policy Scoring Processor shall read severity exclusively from the Policy Store keyed on the rule's CWE and shall discard any severity string carried in legacy rule data. -3. While two enabled rules map to the same CWE, the Policy Scoring Processor shall assign both findings the identical policy-governed severity so that two rules for one CWE can no longer disagree. -4. The Ruleset Store shall support exactly three rule statuses — `enabled`, `shadow`, and `disabled` — where `enabled` fires and scores and reports, `shadow` fires and records outcomes but is excluded from the report and the score, and `disabled` does not fire. -5. When a finding's rule status is `shadow`, the Policy Scoring Processor shall exclude that finding from the reported findings and from the project-score input while still recording its outcome for precision tracking. -6. The Policy Store shall expose, per CWE, an authoritative 0-5 severity plus a static scope flag and a dynamic scope flag, and shall mark dynamic-only CWEs as report-only and never scored by the static tool. -7. If an enabled rule's CWE does not resolve in the Policy Store at startup, then the Scan Runtime shall fail loud and shall not proceed with the scan. -8. The Ruleset Store shall remap the C/C++ buffer-handling rules that reference the unmapped CWE-120 to CWE-121 (severity 5) so that those rules contribute a nonzero penalty to the project score. -9. The Ruleset Store and the Policy Store shall be loaded with the standard library only and shall require no third-party dependency. - -### Requirement 3: Bounded Rule and Policy Adaptation in the Self-Improvement Loop - -**Objective:** As an OpenUltraSAST maintainer, I want the HarnessX meta-harness to adapt rules and policy gates only through two bounded levers, so that the loop can suppress, stage, or raise the bar on noisy rules without ever inventing detectors or rewriting the severity authority. - -#### Acceptance Criteria -1. The Self-Improvement Loop shall expose exactly two new edit levers: a `rule` lever covering rule status, evidence floor, and threshold, and a `policy` lever covering evidence floor, scope, and the score constants `K` and `MIN_SCORE`. -2. The Rule Edit Validator shall reject any loop edit that modifies detection pattern text, so that pattern authoring remains human-pull-request only. -3. The Policy Edit Validator shall reject any loop edit that modifies the authoritative 0-5 CWE severity, so that the severity authority remains upstream-governed. -4. If the loop proposes moving a rule directly from `enabled` to `disabled`, then the Rule Edit Validator shall reject the edit and require `shadow` as a mandatory intermediate staging state. -5. When validating a proposed edit, the Rule Edit Validator shall assert that the edit parses, that `status` is one of `enabled`/`shadow`/`disabled`, that any threshold or evidence-floor tightening or loosening stays within configured bounds, that an added rule does not duplicate an existing rule identifier, and that every referenced CWE resolves in the Policy Store. -6. If the loop re-proposes an edit that was previously reverted without supplying a retry rationale, then the Self-Improvement Loop shall block the re-proposal via the novelty gate. -7. Before accepting any round, the Self-Improvement Loop shall run a replay smoke gate that proves the edited ruleset and policy still load and boot. -8. When any loop edit is rejected during validation, the Self-Improvement Loop shall raise a rule-change validation error and reject the entire round. -9. The Self-Improvement Loop shall record each rule and policy edit, its lever, and its outcome in the journal so that every change is attributable and reversible. - -### Requirement 4: Benchmark-Driven Iteration with a Hard Acceptance Gate - -**Objective:** As an OpenUltraSAST maintainer, I want benchmark results to become actionable per-rule signals that drive the self-improvement loop under a hard acceptance gate, so that detection quality improves automatically while regressions are reverted byte-for-byte. - -#### Acceptance Criteria -1. The Benchmark Harness shall attribute every recorded miss and every recorded false positive to a `rule_id` and shall aggregate per-rule precision and recall in its metrics. -2. The Benchmark Harness shall emit a per-rule recommendation artifact alongside its records, recommending enable, disable, shadow, threshold, or evidence-floor changes per rule. -3. When the orchestrator converts benchmark output to loop input, it shall write recall-side items as miss signals and precision-side items as verifier-reject or false-positive signals into the loop's signal file. -4. After the meta-agent edits rules or policy, the Self-Improvement Loop shall re-run the benchmark and shall accept the round only if aggregate recall is at least 90% AND aggregate false-positive rate is under 10% AND the project score did not regress on the held set AND no unpredicted regression occurred. -5. If a re-run round fails any acceptance condition, then the Self-Improvement Loop shall revert the round and restore the rule and policy configuration byte-for-byte. -6. If a rule drags aggregate precision below the gate, then the Self-Improvement Loop shall auto-shadow that rule rather than delete it, so that recall is preserved while the false-positive cost is removed. -7. When a coverage gap surfaces as a persistent miss, the Self-Improvement Loop shall nominate a new or loosened rule through the signal file while leaving pattern authoring to a human pull request. -8. The Self-Improvement Loop shall treat maximizing the project score as the optimization objective and the recall/FP gate as a separate hard feasibility constraint, so that the loop cannot improve the score by shadowing rules in a way that harms real security. - -### Requirement 5: Project Scores via verycode Policies - -**Objective:** As a security reviewer, I want a single 0-100 project score computed from policy-weighted, reachability-adjusted findings and gated in CI, so that a scan produces a bounded AppSec quality signal alongside its findings. - -#### Acceptance Criteria -1. The Score Model shall compute a single 0-100 project score from policy-governed severity weights multiplied by a reachability multiplier over the reported findings, using the standard library only. -2. The Score Model shall use reachability multipliers keyed exactly on the three emitted reachability values `reachable`, `inferred-file-surface`, and `unknown`. -3. When a finding maps to a CWE that is dynamic-only or unmapped in the Policy Store, the Score Model shall assign that finding a zero score penalty and shall report it as out-of-scope or unmapped rather than scoring it. -4. The Score Model shall exclude findings whose rule status is `shadow` from the score input before computing the project score. -5. The Score Model shall emit a scan-level score artifact containing the project score, the maximum severity, the total penalty, a by-category breakdown, the out-of-scope dynamic-only count, the unmapped CWE list, and the gate verdict, and shall include the score block in the scan manifest. -6. When confirming a false positive, the Calibration subsystem shall lower the finding's effective reachability multiplier rather than delete the rule, so that recall stays high while the score stops over-penalizing. -7. If any reported finding maps to policy severity 5 AND has `reachability_status` equal to `reachable`, then the CI Score Gate shall fail the build. -8. Where the score gate is configured as blocking, the CI Score Gate shall fail the build when the project score is below `MIN_SCORE`; until the score constants are calibrated against the corpus, the CI Score Gate shall compute and report the score advisory-first without blocking. -9. The Score Model and the CI Score Gate shall run in the zero-dependency default install with no HarnessX present. - -### Requirement 6: Detection Gate Non-Regression - -**Objective:** As an OpenUltraSAST maintainer, I want the existing detection gate of at least 90% recall and under 10% false positives on the benchmark corpus to hold across every phase and every change, so that no governance, scoring, runtime, or self-improvement change can silently degrade detection quality. - -#### Acceptance Criteria -1. The CI Pipeline shall evaluate aggregate recall and aggregate false-positive rate on the benchmark corpus on every merge and shall fail the merge if recall is below 90% or the false-positive rate is at or above 10%. -2. While the default ruleset and an empty loop ledger are in effect, the Benchmark Harness shall produce results byte-identical to the pre-feature baseline for the governance and rules-as-data phases. -3. When the self-improvement loop proposes any rule or policy change, the Self-Improvement Loop shall require the recall/FP gate to pass on the re-benchmarked candidate before the change can be accepted. -4. If a HarnessX-backed stage is enabled, then the CI Pipeline shall confirm recall and false-positive rate stay within the configured tolerance of the equivalent zero-dependency baseline. -5. The CI Pipeline shall treat the recall/FP gate as a hard constraint that is never folded into the project-score reward, so that improving the score can never substitute for meeting the detection gate. -6. If any change would reduce recall below 90% or raise the false-positive rate to 10% or above, then the CI Pipeline shall block the change regardless of any project-score improvement. - -### Requirement 7: HarnessX Dependency-Weight Decision - -**Objective:** As an OpenUltraSAST maintainer, I want an explicit dependency-weight decision that keeps the core install zero-dependency and chooses between an optional HarnessX extra and a vendored lean subset based on an audited dependency graph, so that a supply-chain auditing tool never silently inherits an unacceptable transitive dependency footprint. - -#### Acceptance Criteria -1. The Packaging Configuration shall keep the core `dependencies` list empty and shall expose HarnessX only as an optional extra pinned to a specific commit revision. -2. The Runtime shall guard every HarnessX import behind a capability check and shall import HarnessX lazily so that heavy code paths stay cold in the default install. -3. Before adopting the HarnessX execution stages, the maintainer shall run a dependency dry-run audit and shall record the realized transitive dependency graph. -4. If the audited transitive graph pulls a browser-binary, container-SDK, or web-server stack as a mandatory dependency and the maintainers judge it unacceptable for a supply-chain auditing tool, then the Packaging Configuration shall switch to a vendored lean subset of HarnessX under the same extra name with the heavy subtrees removed. -5. While HarnessX is absent, the CI Pipeline shall run a zero-dependency guard test confirming the governance, scoring, and benchmark planes operate with no HarnessX present. -6. Where the project is developed via a submodule, the Runtime setup shall initialize HarnessX non-recursively so that the large reinforcement-learning fork subtree is not pulled. -7. The Packaging Configuration shall keep the vendored lean subset pre-specified so that switching from the optional extra to the vendored subset is a packaging change and not a redesign. diff --git a/.kiro/specs/harnessx-self-improving-rulesets/research.md b/.kiro/specs/harnessx-self-improving-rulesets/research.md deleted file mode 100644 index bb859d5..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/research.md +++ /dev/null @@ -1,191 +0,0 @@ -# Research & Design Decisions - ---- -**Purpose**: Capture the discovery findings and architectural investigation behind the `harnessx-self-improving-rulesets` spec. This is the background-notes companion to `design.md` — it records *what was read*, *what was found in the source*, and *why* the three resolved decisions came out the way they did. `design.md` carries the committed architecture; this file carries the evidence and the rejected alternatives. ---- - -## Summary -- **Feature**: `harnessx-self-improving-rulesets` -- **Discovery Scope**: Complex Integration (adopt an external agent-runtime library + re-architect the in-house rule/policy/scoring planes) -- **Key Findings**: - - HarnessX and OpenUltraSAST (OUS) already both have a thing called a "harness" with "processors" and "events", but they are **incompatible at every load-bearing point**: HarnessX is async-generator / message-window-contract / transcript-state; OUS is sync / declared-`reads`/`writes`-contract / dict-state. There is no drop-in. - - HarnessX's real, reusable value for OUS is concentrated in three places: the **LLM hunter agent loop** (control/tools/observability processors), the **llm-judge verifier** (`build_judge_prompt`/`parse_judge_response`/`_call_judge`), and the **meta-harness self-improvement engine** (journal / novelty / evidence / replay / attribution). Everything else (lab web UI, browser tools, sandboxes, office skills) is unused weight. - - OUS today fuses **detector + severity authority + volume control** into one regex layer (`PATTERN_RULES` in `findings.py`); its self-improvement loop only demotes *ranking* (floored at `0.1`, keyed on `tags[0]`), never the rules; and its benchmark already names the ruleset as the thing to fix (`next_improvement_candidate="rules_or_language_hunter"`) but writes it to an inert JSON. The signals exist; the loop back onto the rules does not. - - The `verycode` policy data (`CWE_Score.tsv`, 167 CWEs, severity 0-5 + static/dynamic scope) is **complete and authoritative**, but the `verycode-score` scoring *algorithm was never implemented* (`// scorer code here` placeholder). The score formula in the design is a labeled, defensible reconstruction. - - OUS today has **zero runtime dependencies** (`dependencies = []`); HarnessX's `dependencies` list is ~30 heavyweight packages (Playwright, Docker SDK, FastAPI/uvicorn, multi-LLM SDKs, office-doc suite) with **no lean extra** to pip-install a subset — a direct hard dependency would invert the project's supply-chain posture for a security tool. - -## Research Log - -### HarnessX core composition & runtime model -- **Context**: We need to know whether OUS's `harness.HarnessRuntime` can be replaced by, or interoperate with, HarnessX's `Harness`, and how an external project builds and runs a HarnessX harness. -- **Sources Consulted**: `/tmp/HarnessX/harnessx/core/{harness,builder,processor,events,runloop,state,contract_check,config_schema,config_store,model_config,trajectory}.py` (read in full); `/home/mc/Source/OpenUltraSAST/src/openultrasast/harness.py`. -- **Findings**: - - Public composition is three objects + one method: `HarnessBuilder.build() -> HarnessConfig`; `ModelConfig.agentic(HarnessConfig) -> Harness`; `await Harness.run(task) -> HarnessResult`. **`HarnessConfig` carries NO model info** — model is always supplied separately via `ModelConfig` (key→provider map, `main` required, non-`main` keys auto-get sub-harnesses). - - `HarnessBuilder` is an immutable fluent factory: `add(proc)`, `slot(**kw)`, `add_tool`, `plugin(source)`, `__or__`/`merge` (collects conflicts → `HarnessConflictError`), `build()` (topo-sorts processors per hook by `_order` then Kahn within a bucket by `_after`). Declarative alt: `build_from_config(dict)` from a flat `processors:` list of `_target_` dicts. - - The run loop (`run_loop`) is a fixed async agent loop: `task_start` → `while True:` (step: `step_start` → `before_model` → call provider → `after_model` → per-tool `before_tool`/exec/`after_tool` → `step_end`) → `task_end`. `get_procs(key)` runs `"*"` (MultiHook) processors on every hook, then hook-specific lists. - - State is **two message tracks** (`raw_messages` append-only, `messages` effective context) + typed `slots: dict[str, StateSlot]` (the nearest analogue to OUS's flat `state` dict, but typed and `_target_`-serializable). `StatefulTrajectory` is a first-class product (`s_t = (z_t, Δz_t, a_t, o_t, e_t, r_t)`), built inline during `run_loop`, with `backfill_rewards` and `to_training_records`/`to_rl_records`. - - Hook-name alignment vs OUS: 5/8 match exactly (`task_start`, `before_model`, `after_model`, `before_tool`, `after_tool`, `task_end`); the gap is `stage_start`/`stage_end` ↔ `step_start`/`step_end`. Semantically an OUS "stage" is a coarse pipeline phase wrapping a callback (`run_stage(name, callback)`), whereas a HarnessX "step" is one model-call iteration — not the same concept. -- **Implications**: A whole-pipeline swap is high risk. The cheapest correct adoption confines HarnessX's async loop to the one OUS stage that *is* an agent loop (the hunter pool) and keeps the sync driver for deterministic stages. This directly motivates Decision D1 (phased seam, not big-bang). - -### The two incompatible processor contracts -- **Context**: OUS's defining feature is a declared `reads`/`writes` per-key state contract. We must know if HarnessX preserves it. -- **Sources Consulted**: `/tmp/HarnessX/harnessx/core/{processor,contract_check}.py`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/harness.py:49-118`. -- **Findings**: - - HarnessX `Processor` is `async def process(event) -> AsyncIterator[Event]` — an async generator that yields transformed events. The recommended base is `MultiHookProcessor` with `on_*` hook methods, an `@on(EventClass)` decorator, and class-attr ordering hints (`_order`, `_after`, `_singleton_group`, `_hook`). There is **no `spec`, no `handles`, no `reads`/`writes`.** - - HarnessX's contract governs **message-window mutation**, not state keys: enforced in `ProcessorChain.process` via `_validate_messages_contract` (system stays at index 0 on `step_start`; `before_model` net length ∈ {0,+1}; appended message must be role `user`; no out-of-window edits) and per-chain `check_post_hook_invariants`. Mode is the env var `HARNESSX_CONTRACT_MODE ∈ {warn(default), strict}` — not a constructor arg. - - OUS `Processor` is **synchronous**, `def handle(event, state: dict) -> dict | None`, with `spec: ProcessorSpec(name, version, handles, reads, writes)`; `_run_processor` enforces *state-key* discipline (reads must pre-exist; writes must be declared; return must be a dict). `ContractMode` is a `"strict"|"warn"` constructor arg. -- **Implications**: The two contracts do not map onto each other. OUS's per-key allow-list (its provenance/audit discipline) has **no home in HarnessX**. To keep that discipline we must reimplement it as a `SlotContractMixin` on our HarnessX processors (`reads_slots`/`writes_slots` class attrs validated in a base `on_*` wrapper), gated by our own `OUS_SLOT_CONTRACT` env. This is Decision D7. - -### HarnessX built-in processors — reuse map -- **Context**: Which HarnessX processors can OUS register directly, which are reusable as patterns/helpers, and which are irrelevant? -- **Sources Consulted**: `/tmp/HarnessX/harnessx/processors/` (control, tools, multi_model, observability, evaluation, memory, context); `/home/mc/Source/OpenUltraSAST/src/openultrasast/cli.py` (`_run_scan`). -- **Findings**: - - **Register-direct only inside the hunter loop** (the one real agent loop): `LoopDetectionProcessor`, `CostGuardProcessor`/`TokenBudgetProcessor`, `ToolFailureGuard`, `SelfVerifyProcessor`, `ToolFilterProcessor`/`ToolWhitelistProcessor`, `ModelRouterProcessor`, `EpisodeMetricsProcessor`/`OTelProcessor`/`CheckpointProcessor`. - - **Reuse as helper functions** for `verify`: the `llm_judge` code (`build_judge_prompt`, `parse_judge_response` with the verdict whitelist, the two-attempt repair loop `_call_judge`) plus the strict-JSON verdict schema (`verdict/confidence/cause/missing_capability`/`lesson`/`missing`). The judge is explicitly told "ground truth is intentionally withheld" — matching OUS's no-oracle scanning. - - **Reuse as pattern** for `calibrate`/`record_calibration`: `MemoryRetrievalProcessor` + `MemoryPolicy` (retrieve/add, should_retrieve/should_store) is the clean interface to refactor the FP ledger toward; `MemoryExtractionProcessor` mirrors distilling verifier outcomes into durable learnings. Storage stays a custom JSON ledger. - - **Keep custom** for every SAST-domain stage: `static_mapping`, `preprocess`, `entry_point_mapping`, `quick_findings`, `report`, `sarif`, `manifest`, and the benchmark recall/FP scoring (HarnessX `EvalResult.score` is a single float — too coarse for the recall/FP gate). -- **Implications**: Adoption is bimodal — register HarnessX classes directly only in the hunter sub-harness; everywhere else port logic as helpers or patterns. This is the backbone of the §3 flow-migration table in `design.md`. - -### HarnessX self-improvement (meta_harness) and the rule-adaptation seam -- **Context**: Can an external signal (a verifier rejection or a benchmark miss) drive a *detection-rule* change through HarnessX's self-improvement loop, and how does it compare to OUS's current calibration? -- **Sources Consulted**: `/tmp/HarnessX/harnessx/meta_harness/{agent,journal,replay,validate_workflow}.py`, `processors/contract_autocheck.py`, `workers/trajectory_digester.py`, `rl/{builder,config,task}.py`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/{calibration,benchmark,findings,rank,cli}.py`. -- **Findings**: - - HarnessX self-improves a *harness* round by round. The unit of change is a `HarnessConfig` (the meta-agent literally rewrites the inner agent's config/processors/templates as files); the unit of memory is a markdown **journal** with per-`## Round N` YAML frontmatter; the unit of safety is a multi-phase **validation gate**. - - The journal classifies every change by a **lever** from `_VALID_LEVERS = {configuration, control, action, instruction}` and tracks `predicted_affected` task_ids plus a `gating_attribution` (`flipped/still_F/regressed/still_T/absent`). `build_context` renders a per-lever **time-decayed Beta posterior** scoreboard (`_DECAY=0.9`) and an explicit "Reverted hypotheses — do not re-propose" list. - - `EvolveValidator.run` enforces validity (canonicalize → contract → diff → replay smoke gate), policy (novelty + evidence, only when the diff is non-empty), advisory. `check_novelty` hard-blocks re-proposing a reverted hypothesis or reverted `(levers, predicted_affected)` signature unless a `retry_rationale` is present. `_evidence` requires a `candidates.md` `## Candidate C-N` cross-referenced by the journal. `run_synthetic_task_smoke_gate` proves the evolved config boots end-to-end. `ContractAutoCheckProcessor` gives in-session `[AUTO-CHECK]` feedback on every authored `processors/*.py`/`tools/*.py` edit. - - **The concrete seam to drive a rule change**: add a lever value to `journal._VALID_LEVERS`; carry the external signal in via a `rule_signals.json` read-context artifact + `_render_task_brief` deliverable; add a validity-phase check (`EvolveValidator._rules`) that raises `StrictValidationError(kind="rule_change")`; add changeset buckets (`compute_changeset` + `journal._CHANGESET_KEYS_ORDER`); extend `contract_autocheck` to classify rule files. The novelty/evidence/replay/attribution machinery then applies for free. - - **OUS by comparison**: detection rules in `findings.py` are static and never modified. Calibration (`calibration.py`) only does `calibrated = max(0.1, calibrated - 0.5)` on a *file's* ranking priority (keyed on `tags[0]`, not `rule_id`) plus a prompt-constraint string — it never disables/tightens/adds a rule, has no novelty gate, no attribution. Benchmark misses produce a `BenchmarkCalibrationRecord` whose `next_improvement_candidate` is hard-coded to `"rules_or_language_hunter"` and written to an **inert** `calibration_records.json` that nothing consumes. The recall side is a fully open loop. -- **Implications**: OUS has exactly the *signals* HarnessX needs (verifier rejections = precision side; benchmark misses = recall side) but no loop closing back onto the rules. The meta_harness is the engine that closes it. This motivates Decision D3 (which levers, and their hard bounds) and the §5/§6 design. - -### The verycode policy & scoring model -- **Context**: The design needs a central CWE→severity authority and a project score. What does `verycode` actually provide? -- **Sources Consulted**: `/tmp/verycode-policies/{README.md,CWE_Score.tsv}`; `gh repo view norandom/verycode-score` (incl. `action.yaml`, `main.go`); `gh repo view norandom/verycode-lib`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/findings.py`. -- **Findings**: - - `CWE_Score.tsv` is the real artifact: 167 data rows, one per CWE, columns `Flaw Category | CWE ID | CWE Name | Flaw Severity (0-5) | Static (X/empty) | Dynamic (X/empty)`. Severity distribution: 12×5, 15×4, 97×3, 31×2, 0×1, 12×0. Scope: 154 Static, 45 Dynamic, 13 Dynamic-only, none neither. (Header column 4 is literally `"Flaw Severity "` with a trailing space.) - - **The scoring algorithm does not exist.** `norandom/verycode-score` is the unmodified `the-gophers/go-action` "Tweeter" template with `// scorer code here` and a `--policy policies/standard.json` flag; `norandom/verycode-lib` is a one-line stub README. The data shape + the "GitHub Advanced Security findings, weighted by CWE severity" descriptions imply a Veracode-style 0-100 pipeline; the design reconstructs the formula and labels it as such. - - **Real coverage gap**: CWE-120 (Classic Buffer Overflow) is **absent from the TSV**, but 3 high-value C/C++ rules use it (`c-unsafe-copy`, `c-unsafe-memory-copy`, `c-unsafe-scanf`). Unmapped → those rules would contribute 0 to the score. CWE-121 (Stack Buffer Overflow) is present at severity 5 and is the precise classic-overflow CWE. - - **Severity mismatch**: OUS rule strings disagree with policy (rules label CWE-78/95/134 "high"; policy rates them 5/critical; rules label CWE-94/502 "high"; policy rates them 3/medium). The policy is the authoritative source. - - The OUS reachability vocabulary is exactly `reachable | inferred-file-surface | unknown` (verified in `findings.py`), which maps cleanly onto a `REACH_MULT = {reachable:1.0, inferred-file-surface:0.6, unknown:0.4}` exploitability multiplier — and that multiplier is the natural FP-calibration knob (lower effective reachability instead of deleting a rule). -- **Implications**: Severity must flow from PolicyStore (`CWE_Score.tsv`), not from the rule. Day-one fixes: remap the 3 CWE-120 rules to CWE-121 (Decision D5); discard rule string-severities (Decision D4). The score is a stdlib-only reconstruction (Decision D8 ships it advisory-first until `K`/`MIN_SCORE` are calibrated). - -### Installability & dependency weight -- **Context**: How does an external project depend on HarnessX, and what does it cost a zero-dependency security tool? -- **Sources Consulted**: `/tmp/HarnessX/pyproject.toml`, `harnessx/__init__.py`, `core/builder.py`, `plugins/{base,loader,discovery,convert}.py`, `examples/{coding,research,assistant}/harness.py`, `extensions/plugins/workflow/plugin.py`, `.gitmodules`; `/home/mc/Source/OpenUltraSAST/pyproject.toml`. -- **Findings**: - - HarnessX is **library-first** and **effectively not on PyPI** (`name="harnessx"`, `version="0.1.0"`, Beta; consumed from Git). `requires-python = ">=3.11"` — exact match with OUS, no floor conflict. Depend via a SHA-pinned Git URL (the version string won't move). - - Custom stages map cleanly onto HarnessX's plugin model: each SAST stage is a `MultiHookProcessor`; group stages + tools + skills into a `HarnessPlugin` mounted via `builder.plugin(...)`; load in-process (instance/class) rather than via directory/`plugin.json`. - - **Dependency-weight is the headline risk.** HarnessX's `dependencies` (always-installed, not extras) is ~30 heavyweight packages: `anthropic`/`openai`/`litellm`/`tiktoken`, `fastapi`/`uvicorn`/`websockets`, `playwright` (ships browser binaries), `docker`/`e2b`, `hydra-core`/`omegaconf`/`structlog`/`loguru`, `opentelemetry-*`, `mcp`, `python-pptx`/`python-docx`/`openpyxl`/`reportlab`/`pdfplumber`. **There is no lean extra** — `pip install harnessx` pulls everything. OUS currently has `dependencies = []`. - - **Mitigating fact**: HarnessX uses **pervasive lazy imports** (heavy modules imported inside functions). So the heavy deps are *installed* but only *imported on demand* — which makes both an optional-extra opt-in and a vendored subset genuinely viable. - - **Submodule caveat**: `.gitmodules` declares a nested `recipe/verl_harnessX/verl` → a large verl RL fork OUS does not need; never `--recursive` init. -- **Implications**: A hard dependency is off the table (it inverts the zero-dep stance and pins to an unmovable Beta). Options narrow to (B) optional extra and (C) vendored lean subset. The lazy-import architecture is what makes B keep heavy code cold by default and makes C a surgical packaging switch. This is Decision D2. - -### Current OUS rule-system problem analysis -- **Context**: Establish, in code terms, why the user's complaint ("rules are too influential and not centrally managed") is true. -- **Sources Consulted**: `/home/mc/Source/OpenUltraSAST/src/openultrasast/{findings,rank,calibration,benchmark,verification,harness,config,cli}.py`. -- **Findings** (six code-level defects): - 1. **Pattern matches dominate output.** In quick mode, every non-comment regex match becomes a finding (`quick_scan_findings`); the only gate is language. Ranking only sorts; verification rejects only on `reachability_status == "unknown"`. A noisy regex sets output volume and FP rate directly — the rule *is* the result. - 2. **Severity is rule-local.** `PatternRule.severity` is a string literal copied straight onto the finding (`_finding_from_match`). Two rules for the same CWE can disagree; nothing reconciles them. - 3. **No central store.** The ruleset is a Python tuple compiled into the package; `config.py` has no rule/policy section. You cannot enable/disable/tune a rule without editing source and cutting a release. - 4. **Calibration demotes ranking, not rules.** `calibrate_ranking` only subtracts `learning.demotion` (default 0.5) from a *file's* `priority`, floored at `MIN_CALIBRATED_PRIORITY = 0.1`, keyed on `tags[0]`. A chronically wrong rule fires at full severity forever; learning targets the wrong object. - 5. **No CWE-policy linkage.** `cwe` is free-text used only for rationale text and loose benchmark matching. No table maps CWE → governed severity/enablement/evidence floor. - 6. **No project-score weighting.** Severity and emission are global constants; the ranking signal (`composite_priority`) only re-sorts. A low-severity crypto finding in a fuzzable network parser is never promoted; a high-severity finding in dead code is never demoted. -- **Implications**: The regex layer is simultaneously **detector, severity authority, and volume knob**, while ranking/verification/calibration/benchmark can only act *around* the rules, never *on* them. The fix is to split those three roles into three owners (rules detect / policy decides severity & scope / score is the optimization target) with a one-directional authority model (governance drives execution; execution emits signals; the loop edits governance under a gate). This is the organizing principle of `design.md`. - -## Architecture Pattern Evaluation - -These are the three "architect angles" that were considered for the overall shape. Each is internally coherent; the synthesis in `design.md` borrows from all three and resolves their three disagreements (D1/D2/D3). - -| Option | Description | Strengths | Risks / Limitations | Notes | -|--------|-------------|-----------|---------------------|-------| -| **Incremental seam** | Keep the OUS sync driver; run HarnessX only inside the `hunter_pool` callback via `asyncio.run`. Rules→data + policy + score added around the existing pipeline. Dependency = optional extra (B) with vendored (C) as escape hatch. Loop edits a `"rule"` lever (status/threshold). | Smallest blast radius; confines the async surface to one stage; keeps the recall/FP gate green per-merge; preserves zero-dep default. | Two harness contracts coexist for a while; `HarnessRuntime` lingers; deterministic stages don't gain HarnessX trajectories yet. | Chosen as the base trajectory (Phases 0-4). | -| **Native rewrite** | Delete `HarnessRuntime` entirely; the *whole* scan becomes a single-step HarnessX harness with `skip_model=True` + slot dataflow; vendored lean subset (C) as the committed default. Loop edits a `"rule"` lever; patterns stay human-PR'd. | Cleanest end-state; one runtime, one contract; full trajectory/journal coverage of every stage; smallest auditable dep graph. | Touches every stage at once — too much blast radius to keep the gate green per-merge; vendoring is permanent fork-maintenance burden. | Demoted to optional Phase 5+; its end-state (single-step harness, retire `HarnessRuntime`) is the long-term target. | -| **Policy-first** | The whole scan runs as processors on a fixed loop; central CWE policy is the centerpiece; loop edits `"rule"` + `"policy"` levers (policy = evidence floors/scope, *not* the 0-5 CWE severity). Dependency = optional extra (B) default, vendored (C) fallback. | Strongest governance story; two levers give the loop the right surface to "stop rules being too influential" without inventing detectors or rewriting severity. | Same whole-pipeline-at-once risk as native if taken literally; needs careful bounds so the loop can't corrupt the ruleset. | Contributed the two-lever model (D3) and the bounded-authority safety rules. | - -## Design Decisions - -The three architect angles agreed on the destination (rules→governed data; severity from a central CWE policy; a gated project score; HarnessX hosting the hunter loop and a rule-adapting self-improvement loop). They disagreed on three load-bearing choices. Each is resolved below with the recall/FP gate as the deciding tie-breaker. (D4-D9 are smaller decisions that follow.) - -### Decision D1: Scope of the HarnessX migration -- **Context**: How much of the scan moves onto HarnessX, and how fast? -- **Alternatives Considered**: - 1. Incremental seam — sync driver stays; HarnessX only in the hunter callback. - 2. Native rewrite — delete `HarnessRuntime`; whole scan becomes a single-step harness immediately. - 3. Policy-first whole-loop — whole scan as processors on a fixed loop. -- **Selected Approach**: **Phased.** Adopt HarnessX directly *at the hunter loop* first (Phase 3); migrate deterministic stages onto a single-step harness only later and optionally (Phase 5+), retiring `HarnessRuntime` only after the seam is proven and the gate has held. -- **Rationale**: The sync→async + contract mismatch is the single highest-risk item in the whole effort. The native rewrite touches every stage at once — too much blast radius to keep ≥90% recall / <10% FP green per-merge. The incremental seam ships the *actual* HarnessX value (agent loop, journal, `StatefulTrajectory`) at exactly the one stage that is already an agent loop. -- **Trade-offs**: Two harness contracts coexist through Phases 3-4 and `HarnessRuntime` lingers; in exchange every phase is independently shippable, test-gated, and reversible. -- **Follow-up**: Golden-output diff gate at each phase; confine `asyncio.run` to the hunter callback; `SlotContractMixin` to preserve key-level discipline. - -### Decision D2: Dependency strategy -- **Context**: A zero-dependency security tool considering a ~30-package agent runtime with no lean extra. -- **Alternatives Considered**: - 1. (A) Hard dependency in main deps. - 2. (B) Optional extra `openultrasast[harnessx]`, SHA-pinned, lazy-imported. - 3. (C) Vendored lean subset under `src/openultrasast/_vendor/harnessx/`. -- **Selected Approach**: **(B) is the shipping default; (C) is a pre-committed, triggered fallback** gated by an explicit `pip install --dry-run` audit before Phase 3. Core `dependencies = []` stays; all HarnessX imports are lazy behind a `_has_harnessx()` check. -- **Rationale**: HarnessX's pervasive lazy imports mean the heavy deps are *installed* but only *imported on demand*, so a clean opt-in extra already keeps heavy code paths cold in the default install — with the least maintenance burden, while tracking upstream. (A) permanently inverts the project's stance and pins to an unmovable Beta `0.1.0`. (C) adds permanent fork-maintenance and should only be paid for when the *installed* graph (not the *imported* graph) is itself unacceptable. -- **Trade-offs**: With B, enabling the extra still inherits the full HarnessX dep set (no lean upstream extra exists); every HarnessX import must be guarded. -- **Follow-up**: Run the dry-run audit and record the realized transitive graph. **Decision rule**: if it pulls Playwright browser binaries / Docker SDK / a FastAPI web stack as *mandatory* transitive deps and maintainers judge that unacceptable, switch to (C) — vendor `harnessx/{core,plugins,processors,bundles,meta_harness,workspace,tools/base}` + the `llm_judge`/`journal`/`validate_workflow`/`agent`/`contract_autocheck` modules, delete `lab/`/browser/sandbox/office subtrees, re-pin a minimal set. The lazy-import design makes the deletion surgical. Never use the recursive submodule path (`recipe/verl_harnessX/verl`). - -### Decision D3: What the self-improvement loop is allowed to edit -- **Context**: The user's core complaint is that rules are "too influential." The loop must be able to *govern* rules without becoming a new way for them to run wild. -- **Alternatives Considered**: - 1. Single `"rule"` lever (status/threshold). - 2. `"rule"` lever; patterns stay human-PR'd (native angle). - 3. `"rule"` + `"policy"` levers, where policy = evidence floors/scope, *not* CWE severity (policy-first angle). -- **Selected Approach**: **Two bounded levers.** `"rule"` = `status` (enabled/shadow/disabled) + evidence-floor + threshold, **never pattern text**. `"policy"` = evidence floor + scope + score constants `K`/`MIN_SCORE`, **never the authoritative 0-5 CWE severity**. Patterns and CWE severities are human-PR-only. -- **Rationale**: This is the precise answer to "stop rules being too influential" — the loop may *suppress / stage / raise-the-bar*, never *invent detectors* or *rewrite the severity authority*. Bounds + mandatory `shadow` staging + auto-revert keep loop authority safe; the meta_harness novelty/evidence/replay/attribution gates apply for free. -- **Trade-offs**: The loop cannot fix a bad pattern itself (it can only `shadow` it and nominate a human PR), and cannot re-weight a CWE — both are deliberate guardrails. -- **Follow-up**: Implement `EvolveValidator._rules`/`._policy` raising `StrictValidationError(kind="rule_change")`; enforce the four "may NOT" bounds (no pattern edits, no 0-5 severity edits, no `enabled→disabled` without `shadow`, no reverted re-proposal without `retry_rationale`). - -### Decision D4: Severity authority -- **Context**: Two rules for the same CWE can disagree on severity today. -- **Selected Approach**: PolicyStore (`CWE_Score.tsv`) is authoritative; `resolve_severity(policy, rule.cwe, target, ranking)` replaces the literal `rule.severity` read; `PatternRule.severity` is removed. Policy always wins; rule string-severities are discarded. -- **Rationale**: One CWE = one governed severity, reconciled centrally; ends rule disagreement. -- **Trade-offs**: Loses the per-rule severity hint (acceptable — it was an authoring accident, not a governed decision). - -### Decision D5: The CWE-120 coverage gap -- **Context**: 3 C/C++ buffer rules use CWE-120, which is absent from `CWE_Score.tsv`. -- **Selected Approach**: Remap `c-unsafe-copy`/`c-unsafe-memory-copy`/`c-unsafe-scanf` to **CWE-121** (Stack Buffer Overflow, severity 5) in the ruleset TOML. -- **Rationale**: CWE-121 is the precise classic-overflow CWE and is already at severity 5; this avoids forking the upstream policy file. Otherwise those high-value rules contribute 0 to the score. -- **Follow-up**: Add a hard CI invariant — every `enabled` rule's `cwe` must resolve in PolicyStore (fail loud at startup); log `unmapped_cwe` in the score artifact; PR genuine gaps upstream to verycode-policies. - -### Decision D6: Response to a confirmed false positive -- **Context**: How does the system react to an FP without hurting recall? -- **Selected Approach**: Lower the finding's effective **reachability multiplier** and/or `shadow` the rule — **never delete it**. -- **Rationale**: The score stops over-penalizing while the rule keeps firing (tracked, just not scored/reported), preserving ≥90% recall and pushing FP <10%. This is exactly the recall/FP balance the project gates on. - -### Decision D7: Where OUS's `reads`/`writes` contract lives -- **Context**: HarnessX has no per-key state contract; OUS's defining discipline must survive. -- **Selected Approach**: Reimplement as a `SlotContractMixin` on our `MultiHookProcessor`s (`reads_slots`/`writes_slots` class attrs validated in a base `on_*` wrapper), gated by `OUS_SLOT_CONTRACT=strict|warn`. -- **Rationale**: The two contracts are incompatible; the mixin preserves provenance/audit discipline inside HarnessX's runtime without trying to bend the message-window contract. - -### Decision D8: Project-score gate rollout -- **Context**: The verycode score formula is a reconstruction (no upstream implementation). -- **Selected Approach**: Ship the score gate **advisory-first** (computed and reported, not blocking); promote to blocking only after `K`/`MIN_SCORE` are calibrated against the benchmark corpus. Score computation is stdlib-only and always on (zero-dep), so CI gating runs in the default install with no HarnessX present. -- **Rationale**: The formula needs corpus calibration before it can safely block a merge; keeping it zero-dep keeps the CI gate independent of the optional harness. - -### Decision D9: RL adoption -- **Context**: HarnessX has an `rl/` subsystem (weight-level improvement) alongside the config-level meta_harness. -- **Selected Approach**: Optional, Phase 5+. -- **Rationale**: Config-level adaptation via the meta_harness is sufficient to close the rule/policy loop; weight-level RL over hunter trajectories is purely additive and not on the critical path. - -## Risks & Mitigations -- **Sync↔async + contract incompatibility (High)** — confine the async surface to the hunter stage via `asyncio.run` in its callback (Phase 3); `SlotContractMixin` for key-level discipline; deterministic stages run `skip_model`; golden-output diff gate at each phase. -- **Dependency posture inversion (High)** — governance/scoring planes stay zero-dep; HarnessX strictly behind the extra + lazy imports + pinned SHA; CI zero-dep guard test; `pip --dry-run` audit before Phase 3; vendored-subset fallback pre-specified. -- **Benchmark-gate regression from rule auto-tuning (High)** — the recall/FP gate is the per-round acceptance test (aggregate + per-rule, hard constraint); `shadow` before `disabled`; novelty + attribution block thrash and reverted re-proposals; auto-revert byte-for-byte. -- **Score gaming the gate (High)** — keep the gate as a *hard constraint*, never part of the reward; track unpredicted regressions; add a score-regression check to acceptance. -- **Meta-agent corrupts the ruleset, re-introducing "too influential" (Medium)** — bounded levers (D3): no pattern rewrites, no severity edits, mandatory `shadow` staging, no `enabled→disabled` jump; recall gate auto-reverts; every change journaled and novelty-checked. -- **Policy completeness / CWE drift (Medium)** — day-one CWE-120→CWE-121 remap; hard CI invariant that every enabled rule's CWE resolves; `unmapped_cwe` logged; treat verycode-policies as authoritative-upstream and PR gaps there. -- **Frozen Beta upstream (Medium)** — pin SHA; document in `VENDOR.md`; keep the meta_harness patch minimal/isolated; rule lever degrades gracefully (absent patch = no rule lever, loop still does ranking demotion); vendored subset removes upstream-instability exposure for depended modules. -- **Slot serialization loss on resume (Medium)** — all scan slots use plain dicts/dataclasses with `_target_` registration; per-slot-type round-trip serialization test (non-serializable content restores as `None` on `wake()`). -- **Two-store coordination drift (Low-Med)** — a single `resolve_severity` owns the rule-status + policy-gate combination; the changeset records both lever edits per round; the benchmark regression suite catches interaction regressions. - -## References -- HarnessX core — `/tmp/HarnessX/harnessx/core/{harness,builder,processor,events,runloop,state,contract_check,model_config,trajectory}.py` — composition/run-loop/contract/state model. -- HarnessX processors — `/tmp/HarnessX/harnessx/processors/{evaluation/llm_judge.py, evaluation/strategies/evaluators/prm.py, observability/{otel_proc,episode_metrics,checkpoint}.py, memory/memory_retrieval.py, control/loop_detection.py}` — reuse map (verify/calibrate/observability). -- HarnessX meta_harness — `/tmp/HarnessX/harnessx/meta_harness/{agent,journal,replay,validate_workflow}.py`, `processors/contract_autocheck.py`, `workers/trajectory_digester.py`; `rl/{builder,config,task}.py` — self-improvement engine + rule-adaptation seam. -- HarnessX packaging — `/tmp/HarnessX/pyproject.toml`, `harnessx/__init__.py`, `plugins/{base,loader,discovery,convert}.py`, `examples/{coding,research,assistant}/harness.py`, `extensions/plugins/workflow/plugin.py`, `.gitmodules` — installability + dependency weight. -- verycode policy — `/tmp/verycode-policies/{CWE_Score.tsv,README.md}`; `gh repo view norandom/verycode-score` (`action.yaml`, `main.go` placeholder); `gh repo view norandom/verycode-lib` (stub) — policy data (complete) vs scoring algorithm (never implemented). -- OUS current system — `/home/mc/Source/OpenUltraSAST/src/openultrasast/{findings,rank,calibration,benchmark,verification,harness,config,cli}.py` — `PATTERN_RULES`, ranking-only calibration (`MIN_CALIBRATED_PRIORITY=0.1`), inert `next_improvement_candidate`, sync `HarnessRuntime`, no rule/policy config section. -- Companion design — `.kiro/specs/harnessx-self-improving-rulesets/design.md` (committed architecture); detection gate per `MEMORY.md`: ≥90% recall / <10% FP on the benchmark corpus. diff --git a/.kiro/specs/harnessx-self-improving-rulesets/spec.json b/.kiro/specs/harnessx-self-improving-rulesets/spec.json deleted file mode 100644 index 38b2cbe..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "feature_name": "harnessx-self-improving-rulesets", - "created_at": "2026-06-22T10:07:18Z", - "updated_at": "2026-06-22T16:45:00Z", - "language": "en", - "phase": "implementation", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true -} diff --git a/.kiro/specs/harnessx-self-improving-rulesets/tasks.md b/.kiro/specs/harnessx-self-improving-rulesets/tasks.md deleted file mode 100644 index df59b9d..0000000 --- a/.kiro/specs/harnessx-self-improving-rulesets/tasks.md +++ /dev/null @@ -1,277 +0,0 @@ -# Implementation Plan - -This plan splits the fused regex layer into three governed owners (rules detect, CWE policy decides severity/scope, project score is the optimization target) and adopts HarnessX behind an optional extra. Tasks are grouped into four phases. Each phase is independently shippable and keeps the ≥90% recall / <10% FP detection gate green. Phases 1-2 (Foundation) and the early Core work stay zero-dependency and byte-identical by default; the HarnessX execution and self-improvement work (Integration) lives behind a lazy, SHA-pinned extra; Validation locks the cross-cutting gates and the dependency-weight decision. - -## Phase 1 — Foundation: Governance & Scoring Planes (zero-dep) - -- [x] 1. Stand up the central CWE policy authority -- [x] 1.1 Vendor the verycode CWE severity data and build the Policy Store loader - - Vendor the verycode CWE-to-severity dataset into the governance plane and parse it with the standard library only, tolerating the trailing-space severity column header. - - Expose, per CWE, an authoritative 0-5 severity, a static scope flag, and a dynamic scope flag; mark dynamic-only CWEs as report-only. - - Observable completion: loading the policy yields the full set of CWE policy entries with correct static/dynamic flags and zero third-party imports, verified by a unit test. - - _Requirements: 2.6, 2.9_ - - _Boundary: PolicyStore_ - -- [x] 1.2 Expose policy-governed severity resolution - - Resolve a finding's severity exclusively from the Policy Store keyed on the rule's CWE, discarding any severity string carried in legacy rule data. - - Guarantee that two findings whose rules map to the same CWE receive identical severity. - - Observable completion: a resolution call returns the policy severity (never a legacy string) and is identical for two rules sharing a CWE, proven by a unit test. - - _Requirements: 2.2, 2.3_ - - _Boundary: PolicyStore_ - -- [x] 1.3 Enforce the fail-loud startup CWE-resolution invariant - - Assert at startup that every enabled rule's CWE resolves in the Policy Store; abort the scan loudly before any work when an enabled rule's CWE is unmapped. - - Observable completion: an enabled rule with an unmapped CWE aborts the scan with a loud error and no findings are produced, proven by an integration test. - - _Requirements: 2.7_ - - _Boundary: PolicyStore_ - - _Depends: 1.1_ - -- [x] 2. Build the project Score Model and CI gates (zero-dep) -- [x] 2.1 Compute the 0-100 policy-weighted, reachability-adjusted project score - - Compute a single 0-100 project score from policy-governed severity weights multiplied by a reachability multiplier over reported findings, using the standard library only. - - Key the reachability multipliers exactly on the three emitted reachability values (`reachable`, `inferred-file-surface`, `unknown`) without redefining the vocabulary. - - Assign a zero score penalty to findings whose CWE is dynamic-only or unmapped, reporting them as out-of-scope or unmapped instead of scoring them. - - Observable completion: a no-penalty corpus scores 100, a single severity-5 reachable finding scores roughly 43, and dynamic-only/unmapped findings contribute zero penalty, proven by boundary unit tests. - - _Requirements: 5.1, 5.2, 5.3, 5.9_ - - _Boundary: ScoreModel_ - - _Depends: 1.1_ - -- [x] 2.2 Emit the scan-level score artifact and manifest score block - - Emit a score artifact containing the project score, the maximum severity, the total penalty, a by-category breakdown, the out-of-scope dynamic-only count, the unmapped-CWE list, and the gate verdict; include the score block in the scan manifest. - - Observable completion: a scan writes a score artifact with every required field and merges the score block into the manifest, verified against the documented artifact schema. - - _Requirements: 5.5_ - - _Boundary: ScoreModel_ - - _Depends: 2.1_ - -- [x] 2.3 Implement the two-condition CI score gate (advisory-first) - - Fail the build when any reported finding maps to policy severity 5 and has reachability `reachable`. - - When configured as blocking, fail the build when the project score is below the minimum; until the score constants are calibrated against the corpus, compute and report the score advisory-first without blocking. - - Observable completion: the hard gate fails on a severity-5 reachable finding while the score gate reports advisory-only by default, proven by gate tests. - - _Requirements: 5.7, 5.8, 5.9_ - - _Boundary: CI Score Gate_ - - _Depends: 2.1_ - -- [x] 2.4 Centralize ruleset/policy/score configuration - - Add configuration entries for the ruleset, policy, and score data locations plus the gate thresholds and emission floors used by later phases. - - Observable completion: the scan reads the score constants and emission thresholds from configuration rather than hardcoded literals, confirmed by a config-driven test. - - _Requirements: 5.8_ - - _Boundary: Packaging Configuration, ScoreModel_ - -## Phase 2 — Core: Rules-as-Data and Benchmark Signals (zero-dep, byte-identical) - -- [x] 3. Convert detection rules into versioned, governed data -- [x] 3.1 Externalize detection patterns into a versioned Ruleset Store - - Load detection rules from versioned data files in which each rule declares a pattern, a CWE, tags, and a loop-controlled status, carrying no rule-local severity field. - - Remove the rule-local severity field from the in-memory rule model so severity can only come from policy. - - Observable completion: the existing pattern rules load from versioned data files with no severity attribute present, proven by a unit test asserting the absence of the field. - - _Requirements: 2.1, 2.9_ - - _Boundary: RulesetStore_ - -- [x] 3.2 Implement the three-status volume control - - Support exactly three rule statuses — `enabled`, `shadow`, `disabled` — where enabled fires/scores/reports, shadow fires and records outcomes but is excluded from report and score, and disabled does not fire. - - Exclude shadow-status findings from the reported set and from the score input while still recording their outcome for precision tracking. - - Observable completion: an enabled rule appears in report and score, a shadow rule is excluded from both yet records an outcome, and a disabled rule never fires, proven by a unit test across all three statuses. - - _Requirements: 2.4, 2.5, 5.4_ - - _Boundary: RulesetStore, ScoreModel_ - - _Depends: 3.1, 2.1_ - -- [x] 3.3 Remap the unmapped C/C++ buffer rules to a scored CWE - - Remap the C/C++ buffer-handling rules that reference the unmapped CWE-120 to CWE-121 so they resolve in policy and contribute a nonzero penalty. - - Observable completion: the remapped buffer rules resolve to a severity-5 policy entry and produce a nonzero score penalty, verified by a unit test. - - _Requirements: 2.8_ - - _Boundary: RulesetStore_ - - _Depends: 3.1, 1.1_ - -- [x] 3.4 Wire findings and severity resolution through the governance plane - - Pass the loaded ruleset into the finding-detection stage instead of reading an implicit module-level rule list, and resolve every finding's severity through the Policy Store. - - Apply emission thresholds so not every regex match becomes a reported finding, replacing the old "rule is its own volume control" behavior. - - Observable completion: a scan produces findings with policy-resolved severity and shadow findings filtered out, while the default ruleset keeps benchmark output byte-identical to the pre-feature baseline. - - _Requirements: 2.2, 2.3, 2.5, 6.2_ - - _Boundary: PolicyScoringProcessor, RulesetStore, PolicyStore_ - - _Depends: 3.1, 3.2, 1.2_ - -- [x] 4. Make benchmark output an actionable per-rule signal -- [x] 4.1 Attribute misses and false positives to rules - - Attribute every recorded miss and every recorded false positive to a rule identifier and aggregate per-rule precision and recall in the benchmark metrics. - - Observable completion: a benchmark run reports per-rule precision/recall and every miss and false positive carries a rule identifier, verified by a metrics test. - - _Requirements: 4.1_ - - _Boundary: Benchmark Harness_ - - _Depends: 3.1_ - -- [x] 4.2 Emit a per-rule recommendation delta artifact - - Replace the inert improvement-candidate stub with a per-rule recommendation artifact recommending enable, disable, shadow, threshold, or evidence-floor changes per rule. - - Observable completion: a benchmark run writes a recommendation artifact with per-rule suggested actions alongside its records, verified against the documented delta schema. - - _Requirements: 4.2_ - - _Boundary: Benchmark Harness_ - - _Depends: 4.1_ - -- [x] 4.3 Introduce the loop-owned ledger and re-key calibration to rules - - Add the loop-owned overlay ledger that records per-rule status, evidence floor, threshold, and precision, applied over the ruleset at load time. - - Re-key calibration from the first-tag key to the rule identifier and write status flips into the ledger rather than only demoting ranking. - - Observable completion: an empty ledger leaves behavior byte-identical to the prior phase while a ledger entry overlays a rule's status at load, proven by an integration test. - - _Requirements: 2.4, 5.6, 6.2_ - - _Boundary: RulesetStore, Calibration subsystem_ - - _Depends: 3.1, 3.2_ - -- [x] 4.4 Convert false-positive confirmation into a reachability knob - - When confirming a false positive, lower the finding's effective reachability multiplier rather than delete the rule, so recall stays high while the score stops over-penalizing. - - Observable completion: confirming a false positive lowers the finding's effective reachability multiplier and the rule still fires, proven by a calibration test. - - _Requirements: 5.6_ - - _Boundary: Calibration subsystem, ScoreModel_ - - _Depends: 4.3, 2.1_ - -## Phase 3 — Integration: HarnessX Execution & Self-Improvement (optional extra) - -- [x] 5. Gate and package the HarnessX dependency-weight decision -- [x] 5.1 Run the dependency dry-run audit and record the transitive graph - - Before adopting the HarnessX execution stages, run a dependency dry-run audit and record the realized transitive dependency graph. - - If the audited graph pulls a browser-binary, container-SDK, or web-server stack as a mandatory dependency and it is judged unacceptable, select the pre-specified vendored lean subset under the same extra name with the heavy subtrees removed. - - Observable completion: the audit produces a recorded transitive-dependency graph and a documented accept-extra-or-vendor-subset decision before any HarnessX execution code is wired in. - - _Requirements: 7.3, 7.4, 7.7_ - - _Boundary: Packaging Configuration_ - -- [x] 5.2 Configure HarnessX as a SHA-pinned optional extra with a capability guard - - Keep the core dependency list empty and expose HarnessX only as an optional extra pinned to a specific commit revision; initialize any submodule non-recursively so the large RL fork subtree is not pulled. - - Guard every HarnessX import behind a capability check and import it lazily so heavy code paths stay cold in the default install. - - Observable completion: the default install resolves with an empty core dependency list and HarnessX symbols are only imported after the capability check passes, verified by an import-cold test. - - _Requirements: 7.1, 7.2, 7.6_ - - _Boundary: Packaging Configuration_ - - _Depends: 5.1_ - -- [x] 6. Adopt HarnessX at the hunter loop and verifier -- [x] 6.1 Run the hunter pool stage on a real HarnessX agent loop - - Run the hunter pool stage on a real HarnessX run loop with the control, tool, and observability processors registered in the hunter sub-harness, invoked from the existing hunter-pool stage callback. - - Confine all asynchronous execution to this single stage so the deterministic stages remain synchronous. - - Observable completion: with the extra installed, the hunter pool produces per-target trajectories from a real agent loop while all other stages stay synchronous, verified by an integration test. - - _Requirements: 1.1, 1.4_ - - _Boundary: HxScanOrchestrator_ - - _Depends: 5.2_ - -- [x] 6.2 Map stage events and identifiers onto HarnessX hooks - - Map the existing stage-start/stage-end events to the HarnessX step-level hooks and the scan identifier to the HarnessX run identifier without losing trace continuity. - - Observable completion: a HarnessX-backed run preserves trace continuity with stage events mapped to step hooks and the scan id mapped to the run id, verified by inspecting the emitted trace. - - _Requirements: 1.7_ - - _Boundary: HxScanOrchestrator_ - - _Depends: 6.1_ - -- [x] 6.3 (P) Run deterministic stages as model-disabled HarnessX processors with slot contracts - - Run each deterministic stage under HarnessX with model invocation disabled so no deterministic stage incurs a model call. - - Preserve each stage's declared read/write slot allow-list and validate slot access on every lifecycle hook. - - Observable completion: deterministic stages execute under HarnessX with no model calls and a slot-contract violation is raised on undeclared slot access, proven by a slot-contract test. - - Implemented zero-dep in `slot_contract.py` (`SlotContractMixin`/`SlotPipeline`, validates reads/writes per lifecycle hook, enforces `skip_model`) + `stage_processors.py` (the deterministic stages as `skip_model` slot processors; `run_quick_pipeline` is byte-identical to the quick path). Per "bridge, do not map", the sync `HarnessRuntime` stays the top-level driver and these slot processors are the model-disabled representation; `host_under_harnessx` is the lazy, capability-gated bridge that disables the model on HarnessX's `BeforeModelEvent.skip_model`. Module names differ from the design's `harness/{slot_contract,processors}.py` because `harness` is a module, not a package. Tests: `tests/test_slot_contract.py`. - - _Requirements: 1.5, 1.6_ - - _Boundary: HxScanOrchestrator_ - - _Depends: 6.1_ - -- [x] 6.4 (P) Add the LLM-judge verifier with a structural fallback - - Produce verification verdicts using the HarnessX llm-judge helpers when the extra is present, mapping the strict-JSON verdict to a verification result. - - Retain the existing structural verifier (which rejects on unknown reachability) verbatim as the zero-dependency fallback. - - Observable completion: with the extra present the judge produces verdicts and with it absent the structural verifier produces equivalent verdicts, proven by a paired verifier test. - - _Requirements: 1.2_ - - _Boundary: VerifyProcessor_ - - _Depends: 5.2_ - -- [x] 6.5 Fall back gracefully when a HarnessX capability is unavailable - - When a HarnessX-backed stage is enabled but its capability is unavailable at runtime, fall back to the existing equivalent stage and record the degradation in the scan manifest. - - Observable completion: requesting a HarnessX stage whose capability is missing runs the existing equivalent and records a degradation entry in the manifest, proven by a fallback test. - - _Requirements: 1.8_ - - _Boundary: HxScanOrchestrator_ - - _Depends: 6.1, 6.4_ - -- [x] 7. Stand up the bounded self-improvement loop -- [x] 7.1 Add the two bounded edit levers - - Expose exactly two new edit levers: a rule lever covering rule status, evidence floor, and threshold, and a policy lever covering evidence floor, scope, and the score constants. - - Observable completion: the loop's valid-lever set includes exactly the rule and policy levers and rejects any edit outside their declared fields, verified by a lever test. - - _Requirements: 3.1_ - - _Boundary: EvolveValidator, improve/evolve.py_ - - _Depends: 5.2_ - -- [x] 7.2 Enforce the rule/policy edit safety bounds in the validator - - Reject any loop edit that modifies detection pattern text and any edit that modifies the authoritative 0-5 CWE severity. - - Reject moving a rule directly from enabled to disabled, requiring shadow as a mandatory intermediate staging state. - - Assert that each edit parses, that status is one of the three literals, that threshold/evidence-floor changes stay within configured bounds, that an added rule does not duplicate an existing identifier, and that every referenced CWE resolves in policy; raise a rule-change validation error and reject the whole round on any failure. - - Observable completion: each prohibited edit (pattern change, severity change, enabled-to-disabled jump, duplicate identifier, unmapped CWE, out-of-bounds threshold) is rejected and rejects the round, proven by validator unit tests. - - _Requirements: 3.2, 3.3, 3.4, 3.5, 3.8_ - - _Boundary: EvolveValidator_ - - _Depends: 7.1, 1.1_ - -- [x] 7.3 Add the novelty, replay, and journal gates - - Block re-proposing a previously reverted edit that lacks a retry rationale via the novelty gate. - - Run a replay smoke gate before any benchmark re-run proving the edited ruleset and policy still load and boot. - - Record each rule and policy edit, its lever, and its outcome in the journal so every change is attributable and reversible. - - Observable completion: a reverted re-proposal without rationale is blocked, a non-booting edited ruleset is rejected before re-benchmark, and every edit appears in the journal with its lever and outcome, proven by gate tests. - - _Requirements: 3.6, 3.7, 3.9_ - - _Boundary: EvolveValidator, improve/evolve.py_ - - _Depends: 7.2_ - -- [x] 7.4 Convert benchmark deltas into loop signals - - Convert the per-rule recommendation delta and false-positive learnings into a loop signal file, writing recall-side items as miss signals and precision-side items as verifier-reject or false-positive signals. - - Observable completion: a benchmark run produces a signal file whose recall-side entries are miss signals and precision-side entries are verifier-reject/false-positive signals, verified against the documented signal schema. - - _Requirements: 4.3_ - - _Boundary: improve/evolve.py_ - - _Depends: 4.2, 7.1_ - -- [x] 7.5 Implement the hard acceptance gate with byte-for-byte revert - - After the meta-agent edits rules or policy, re-run the benchmark and accept the round only if aggregate recall is at least 90% AND false-positive rate is under 10% AND the project score did not regress on the held set AND no unpredicted regression occurred. - - On any failed acceptance condition, revert the round and restore the rule and policy configuration byte-for-byte. - - Observable completion: a round that regresses recall is reverted and the configuration is restored byte-for-byte, while a compliant round is accepted and persisted, proven by a self-improvement integration test. - - _Requirements: 4.4, 4.5, 6.3_ - - _Boundary: improve/evolve.py_ - - _Depends: 7.3, 7.4, 4.1_ - -- [x] 7.6 Auto-shadow precision-draggers, nominate coverage gaps, and separate the objective - - Auto-shadow a rule that drags aggregate precision below the gate rather than delete it, preserving recall while removing the false-positive cost. - - Nominate a new or loosened rule through the signal file when a coverage gap surfaces as a persistent miss, leaving pattern authoring to a human pull request. - - Treat maximizing the project score as the optimization objective and the recall/FP gate as a separate hard feasibility constraint that is never folded into the score reward. - - Observable completion: a deliberately noisy rule is auto-shadowed (not deleted) when it drags precision, a persistent miss emits a rule nomination, and the loop cannot raise the score by shadowing rules that breach the gate, proven by integration tests. - - _Requirements: 4.6, 4.7, 4.8_ - - _Boundary: improve/evolve.py_ - - _Depends: 7.5_ - -## Phase 4 — Validation: Cross-Cutting Gates & Non-Regression - -- [x] 8. Lock the detection-gate non-regression checks into CI -- [x] 8.1 Enforce the per-merge recall/FP detection gate - - Evaluate aggregate recall and aggregate false-positive rate on the benchmark corpus on every merge and fail the merge if recall is below 90% or the false-positive rate is at or above 10%. - - Treat the recall/FP gate as a hard constraint that is never folded into the project-score reward, blocking any change that breaches the gate regardless of score improvement. - - Observable completion: a CI run fails the merge when recall drops below 90% or false-positive rate reaches 10%, and a score improvement cannot unblock a gate-breaching change, proven by pipeline tests. - - _Requirements: 6.1, 6.5, 6.6_ - - _Boundary: CI Pipeline_ - - _Depends: 4.1_ - -- [x] 8.2 (P) Assert byte-identical benchmark baseline for the zero-dep phases - - With the default ruleset and an empty loop ledger, assert the benchmark output is byte-identical to the pre-feature baseline for the governance and rules-as-data phases. - - Observable completion: a golden-diff test confirms byte-identical benchmark output with the default ruleset and empty ledger. - - _Requirements: 6.2_ - - _Boundary: Benchmark Harness, CI Pipeline_ - - _Depends: 3.4, 4.3_ - -- [x] 8.3 (P) Confirm HarnessX-backed stages stay within detection tolerance - - When a HarnessX-backed stage is enabled, confirm recall and false-positive rate stay within the configured tolerance of the equivalent zero-dependency baseline. - - Observable completion: a CI run with a HarnessX-backed stage enabled keeps recall/FP within the configured tolerance of the baseline, proven by a tolerance test. - - _Requirements: 6.4_ - - _Boundary: CI Pipeline, HxScanOrchestrator_ - - _Depends: 6.1, 8.1_ - -- [x] 8.4 (P) Add the zero-dependency guard test - - Run a guard test confirming that with HarnessX absent the governance, scoring, and benchmark planes operate and CI gating runs in the default install. - - Observable completion: the guard test passes with HarnessX uninstalled and fails if any governance/scoring/benchmark path imports HarnessX. - - _Requirements: 7.5_ - - _Boundary: CI Pipeline_ - - _Depends: 8.1_ - -- [x] 9. Validate scan-pipeline integrity and serialization -- [x] 9.1 (P) Verify capability fallback and unchanged sync-driver behavior - - Confirm that with the extra absent, quick and standard scans execute through the existing sync driver with behavior unchanged from the pre-adoption baseline, and that a requested-but-unavailable HarnessX stage records a degradation. - - Observable completion: an extra-absent scan matches the pre-adoption baseline and a missing-capability request records a manifest degradation, proven by an end-to-end fallback test. - - _Requirements: 1.3, 1.8_ - - _Boundary: HxScanOrchestrator, CI Pipeline_ - - _Depends: 6.5_ - -- [x] 9.2 (P) Verify slot serialization round-trips on resume - - Confirm each scan slot type serializes and restores without loss after a resume, guarding against non-serializable slots restoring as empty. - - Observable completion: a per-slot-type round-trip test passes for every scan slot after a resume cycle. - - Done for the current artifact-persistence model (findings/ranking/file-target/verification/score round-trip + the real findings.json reload path in `tests/test_scan_pipeline_integrity.py`). The HarnessX processor slot-contract serialization variant remains deferred with task 6.3. - - _Requirements: 1.5_ - - _Boundary: HxScanOrchestrator_ - - _Depends: 6.3_ diff --git a/.kiro/specs/learning-harness/brief.md b/.kiro/specs/learning-harness/brief.md deleted file mode 100644 index 17a9f3b..0000000 --- a/.kiro/specs/learning-harness/brief.md +++ /dev/null @@ -1,113 +0,0 @@ -# Brief: learning-harness - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Problem - -The tool is a hand-authored static analyzer with a learning loop bolted on the side, and the loop cannot learn. Three deliberate choices cap it: closed vocabularies (sinks, sources, obligation kinds) that grow only by writing a new spec; a corpus that is the only teacher and a shape that is the only lesson; and a ban on retaining anything the model discovers. Results have ceilings by construction, the loop trains on its own test set, and there is no failure analysis feeding the next round. Detection logic is ~16% of the source and the other 84% cannot say whether the 16% works. - -Measured 2026-09-06 on the vibe-py holdout (17 vendored pairs), the existing tool hunter (Sonnet 5 through OpenRouter, 4 steps, tools `read_file`, `grep_repo`, `find_refs`, prompted from file-level hotspots, no learning of any kind) against the overlay: - -| scorer | published number | class-aware, identical twins excluded (n = 14) | -|---|---|---| -| tool hunter | 0/17 correct, Youden −0.588 | **12/14 correct, Youden +0.857, 0 leaks** | -| overlay (facts + shapes) | 5/17 correct, Youden +0.294 | 5/14 correct, Youden +0.357 | - -The published hunter number is a scoring artifact: the pair scorer counts a hunter finding only when the finding text contains the literal CWE id and the prompt never asks for one, so 24 of 25 vulnerable-side findings that sit inside the labeled function and name the labeled bug were thrown away. The 24 "leaks" split into 3 findings on byte-identical twins and 21 findings outside the labeled function on the trap twin, most of them real defects in a deliberately vulnerable repository. Raw runs: `benchmarks/measurements/2026-09-06-vibe-py-holdout-hunter-*.json`. - -Three consequences follow. - -1. **The model with tools is already the best detector we own, without any learning.** The roadmap thesis ("recall from the model with tools plus proof; precision and silence from static checks") is confirmed, and the harness has been optimizing the wrong component. -2. **Detection, silence and leak are meaningless without a class.** Every scorer, lever and gate uses text tokens (CWE strings, sink names) as a proxy for class; the corpus's own labels are 32 free-text strings across 7 slices, and 130 of 176 vfc rows say `other`. -3. **Improvements without a class dimension oscillate.** A change that helps one class and hurts another reads as no change on an aggregate Youden; the loop keeps or reverts it for the wrong reason and the system degrades in ways nobody predicted. This is the maintainer's observation and it is what the corpus-seeded lever does today. - -Repairs already measured and required regardless: the mechanism lever seeds candidates from holdout pairs (holdout Youden +0.059 with every pair teaching, −0.059 with the train split only); five vibe-py twins are byte-identical; TypeScript has no grammar; absence excerpts carry no registration context; unit fixtures are easier than the corpus rows they stand in for. - -## What others learned - -Full sourced notes in `research.md` (four threads, 2026-09-06). The findings that change this design: - -**Cyber reasoning systems (AIxCC finals, Big Sleep, XBOW, Glasswing).** The routing key that worked in every finalist was mode × language × sanitizer or CWE family, aligned to what the oracle can confirm; nobody routed on fine CWE. Only proof survives: every top system gates a report on an executed oracle, the accuracy multiplier decided the ranking (Atlantis 0.9999 versus 0.9044), and XBOW keys a non-AI validator per vulnerability class that alone decides success. RoboDuck's cheap logprob classifier (~$0.001) sending the top 20% to a $0.50 agent is the closest public analogue of an auto-classifier front end. Ensembles beat selection when the winner is unpredictable, with attribution done per agent after the fact. Validated is not correct: 38–46% of automatically accepted patches were semantically wrong. Hallucination controls are structural (forced tool calls, curated tools instead of bash, symbol resolution). Cross-run learning is nearly absent in the field, and the one trained component reported catastrophic forgetting across projects. Stability and cost decided more than cleverness. - -**Self-improving harnesses (GEPA, DSPy, AlphaEvolve, DGM, Self-Harness, SkillOpt, AHE, Meta-Harness).** Track wins per instance or class, never the aggregate: GEPA's per-instance Pareto archive doubled the gain of greedy selection. Accept a change only when train and holdout are both non-negative and one is strictly positive; reject ties. Meta-agents predict the class they help at 33.7% precision and the class they hurt at 11.8% ("regression blindness"), so a cross-class sweep cannot be optional. Two-stage evaluation is where sample efficiency comes from. Prompt bloat is the overfit signature; cap length. Structural edits (tools, checklists, memory) transfer across model families, prose edits regress. Self-diagnosis confabulates (0 of 121 reflections named the real cause; structured trajectory facts reached 86%). Hide the checker and the gold labels from the editable surface. One change, one class per round. Harness-edit quality is flat across model tiers while detector quality is not: a cheap meta-agent, spend on the detector. - -**Datasets and taxonomies.** Deduplicate by normalized hash before splitting and split by repository, chronologically. Report pair-wise outcomes (pair-correct, both-vulnerable, both-benign, reversed) and a directional-bias index, not F1; models that flag both sides win F1 while losing pair accuracy. Match on location and class with hierarchical partial credit and the maximum penalty for a hallucinated class. Fine CWE classification from code is unsolved (top-1 ≤ 14.7%, most errors lateral siblings); coarse families are workable (88 CWEs collapsed to 12 gave per-class experts +12.8% F1, with a router right 63.8% of the time and F1 0.18 when wrong). Specialists collapse without hard negatives. Verdicts flip on renames and whitespace; single-shot evaluation missed 42% of real changes, so evaluate with K ≥ 3 runs and a reliable-change test. Function-level pairs lose context, the same defect as our context-free absence excerpts. - -**Routing and regression control.** Learned routers plateau 10–20 points below an oracle and can lose to random; measure router accuracy on its own against random and oracle. Route to several experts with an explicit `unknown`, never argmax. Slice everything; aggregate gains of +2 points have masked 28–39% of items getting worse. Freeze per-class configurations to prevent interference; skill loops driven by self-feedback alone drift, external feedback does not. Production gates are conjunctive per slice with relative thresholds against the last blessed baseline; no public numeric negative-flip budget exists, so the defensible budget is the measured seed-noise floor per class. - -## Desired Outcome - -A harness that improves a **set of classified detectors** round by round, where every change is attributed to the class it was meant to move, every unpredicted regression in another class counts against it, only what a verifier can confirm is reported above suspicion, and nothing is published without its unscorable denominator and its noise floor. - -- **Auto-classifier as routing key.** A closed, versioned taxonomy of roughly 8–12 families derived from the CWE hierarchy and aligned to what our verifiers can confirm (sandbox recipe, static corroboration, entry-point and flow tools). Multi-label with an explicit `unknown`. Three tiers: deterministic static signals first, a cheap model second with logprob averaging, `unknown` third, each answer recording its tier. Measured on its own: agreement with maintainer labels on reviewed pairs with hierarchical partial credit, accuracy against a random router and an oracle, confusion matrix per round. Disagreements are the review queue. -- **Per-class detectors, frozen between rounds.** One hunter configuration per family: length-capped prompt, skills, curated tools with symbol resolution (never bash), step and cost budget, evidence checklist, hard negatives, known counterexamples. A region routes to every family the classifier admits plus a generalist. Round zero clones the current hunter into every family so the baseline is controlled; K ≥ 3 runs per class establish the seed-noise floor that becomes that class's regression budget. -- **Class-aware, pair-wise scorer.** Detection = a finding in the labeled function whose class matches the label, with hierarchical credit and the maximum penalty for a hallucinated class. Silence = no finding of the labeled class in the labeled function on the fixed side. Outcomes as pair-correct, both-vulnerable, both-benign, reversed, plus a directional-bias index, negative flips per item against the last blessed round, and a fixed-FPR view. Unscorable rows (identical twin, unsupported language, unresolved label, missing context) listed by reason and removed from the denominator, never deleted. Dedup by normalized hash before splitting; split by repository, chronologically. -- **Only proof is reported above suspicion.** Class-keyed verifiers decide: sandbox recipes, static corroboration, entry-point, flow and obligation tools. A hunter claim without a verifier stays `suspicion` and is scored as such. Verifier code and gold labels sit outside every editable surface, and anything reported as proven gets a second independent judge. -- **Learning unit = structural change, journaled.** A round's output is a tool, a checklist, a counterexample, a memory entry or a bounded prompt edit for one class, drafted from structured failure facts (which sink, which line, which check fired), never from free-form self-critique. Accepted only when train and holdout are both non-negative with one strictly positive and no other class falls below its budget; ties rejected; rejected edits kept in a buffer the proposer sees; the proposer never sees the holdout. HarnessX's `MetaAgent.evolve` and journal (`hypothesis_id`, `levers`, `predicted_affected`, precision discounted by `regressed_unpredicted`) is the round primitive; the tool supplies tasks keyed by class, trajectories, the evaluator and the routing. -- **Honest fitness.** Split enforced by construction; two-stage evaluation (minibatch gate, the class's full holdout, the cross-class sweep); a per-class archive of which candidate wins which instance rather than one champion; the roadmap rewritten from one command with the unscorable denominator and the noise floor beside every number; a per-round cost cap where an over-budget round is reverted and recorded. - -Principle for the roadmap: **classify first, verify before reporting, improve one class at a time under a fitness that punishes what it did not predict.** - -## Approach - -1. **Class-aware pair scorer and unscorable rows.** Pair-wise outcome vocabulary, location-and-class matching with hierarchical credit, directional-bias index, per-item negative flips, K-run reliable-change test, unscorable reasons out of the denominator, normalized-hash dedup, repository-level split. Re-publish every slice from one command. Smallest change, biggest number. -2. **Taxonomy and auto-classifier.** Family taxonomy versioned in one file with its CWE mapping and its verifier per family; three-tier classifier; its own measurement report; the vfc slice's 130 `other` rows and the 32 free-text classes are its first job, disagreements queued for review. -3. **Per-class hunter configurations.** Clone the current hunter into every family; curated tools with symbol resolution; length-capped prompts; per-family evidence checklist and hard negatives; the classifier routes a region to every admitted family plus the generalist; findings carry their class, detector and verifier. -4. **Verifiers keyed by class.** Sandbox recipes, static corroboration, entry-point, flow and obligation tools exposed to the hunter and to the scorer; a claim rises above `suspicion` only through a verifier; a second independent judge for anything proven. -5. **Split enforcement and lever refusal.** Every exporter and lever takes the train split by construction and refuses candidates touching a holdout pair; re-measure `corpus-seeded-mechanisms` and rewrite its roadmap row. -6. **Evolve loop on HarnessX.** Tasks = classified pairs (`RLTask.task_type` = family); trajectories from journaled hunter runs; evaluator = the class-aware scorer; one structural change for one class per round with `predicted_affected` set; the acceptance rule and cross-class sweep above; per-class Pareto archive; rejected-edit buffer; reflector blind to the holdout; per-round cost cap; the journal's attribution table is the round report and the daily benchmark. -7. **Corpus repairs the loop needs.** TypeScript grammar in the semantic extra; absence rows re-harvested with `handler_context`; identical twins marked `known_limit:identical_twin`; a fixture-difficulty check that fails when a unit fixture is strictly easier than the corpus rows its test claims to cover; a post-cutoff slice for contamination. -8. **Operational floor.** Persisted logs and trajectories, budget caps tested on the failure path, redaction on every prompt and journaled trajectory, cost per point published per round. - -## Scope - -- In: items 1–8; roadmap rewrite from one command; a closure decision on `authorization-obligations` 4.3, 5 and 6. -- Out: new static shape families; hand-written facts beyond what a round proposes and validates; changing the evidence ladder or dispositions; any merge gate on non-local slices; vendoring new pairs beyond re-harvesting existing recipes; fine-tuning model weights. - -## Boundary Candidates - -- Taxonomy and classifier (new, owns class assignment and its measurement) vs pair labels (maintainer-owned reference) -- Per-class detector configurations (new, evolved, frozen between rounds) vs static analyzers (unchanged internally, exposed as tools and verifiers) -- Class-aware scorer (replaces the text-token rule) vs catalog schema (one additive `unscorable` reason, one `class` field) -- Evolve loop (new, HarnessX) vs existing bounded levers (rules, policy, mechanisms; kept, subordinated to split, class and budget) - -## Out of Boundary - -- Taint semantics, walker, dominance internals (`guard-dominance-regime`, `overlay-ir-completeness`) -- Call graph and path records (`reachability-flow-model`), except as a future hunter tool -- Obligation checker internals (`authorization-obligations`); its corpus tasks move here, its lever waits for the split - -## Upstream / Downstream - -- Upstream: `pair-corpus-honesty` (labels, tiers, splits, `known_limit`), `corpus-seeded-mechanisms` (exporter, LOO, lever), `tool_hunter` and `provider/openrouter`, `harness_ext`, HarnessX 0.1.0 (`meta_harness`, `rl`, `tracing.journal`), `tree-sitter-overlay-extra`, the sandbox and regress recipes. -- Downstream: every roadmap number; `harnessx-self-improving-rulesets` (its evolve step is this spec); `authorization-obligations` 4.3/5/6; scan reports (findings carry class, detector and verifier). - -## Existing Spec Touchpoints - -- Extends: `pair-corpus-honesty` Req 1 (scorer) and Req 9/10 (row classification); `corpus-seeded-mechanisms` Req 4/5 (split by construction); `three-stage-scan` MAP (per-class hunters as proposers); `propose-adjudicate-prove` (verifiers keyed by class); `harnessx-self-improving-rulesets` (the evolve loop lands here). -- Folds in: the retired `closed-loop-integrity` brief. -- Re-homes: `authorization-obligations` 4.3 and the corpus half of 6.1; holds 5. - -## Constraints - -Core `dependencies = []`; HarnessX and grammars stay optional extras; every loop step degrades to a recorded reason without them. No row deleted; unscorable rows keep a reason. Labels stay maintainer-owned: the classifier proposes, disagreements queue for review, and a class change on a reviewed pair is a reviewed edit. Model-authored skills, tools and prompt edits are retained only after held-out validation under the acceptance rule and are journaled with attribution; nothing a round proposes reaches a scan without its class holdout and the cross-class sweep. Split enforced by construction. Verifier code and gold labels outside every editable surface. Prompt length capped per class. Gates (`gate`, `map_gate`, `pair_gate`) stay byte-identical. Redaction on every prompt and trajectory. Per-round cost cap with revert. - -## What this project is for (maintainer, 2026-09-06) - -The target is to catch the taxonomy families below **reliably** on real-world application code, including absence bugs in vibe-coded services (an authorization check that was never written), with an emphasis on dangerous, low-hanging fruit. It is not a bug-finder for JavaScript interpreters, WebGL renderers or browser engines, and it is not a memory-safety fuzzer: memory bugs in C and C++ are a different game played by specialized systems (AIxCC-class CRSs, Big Sleep) that this project learns from and does not replicate or compete with. The balance to hold is between two failure modes, being useless on actual code and being blind to forgotten checks in generated code; both are in scope, engine and renderer bugs are not. The vfc C slice therefore stays a measured, non-gating slice with one coarse memory-safety family and no crash oracle of our own. - -## Decisions (maintainer, 2026-09-06) - -1. **Taxonomy spine.** Ten families keyed to danger and to a verifier we can run, with CWE and the mechanism vocabulary as attributes, never the routing key: injection into a query or command (SQL, command, code, template); file path from user input; deserialization of untrusted data; access control (IDOR, missing guard, identity from request body; the obligations checker is its verifier); output encoding (XSS, template to HTML); untrusted destination (server-side request forgery and open redirect, which share the shape of untrusted input choosing where a request goes and account for 34 corpus rows); prototype pollution and unsafe object merge; permissive configuration and secrets (CORS, debug, cookie flags, weak crypto, hardcoded credentials); memory safety as one coarse family for the vfc slice only; unknown. Multi-label routing. -2. **Regression budget.** Measured, not chosen: round zero runs the cloned hunter K = 5 times per family at temperature 0 over the vendored corpus; the per-family negative-flip rate between seeds is the budget. A round that exceeds it in any family is rejected regardless of its target gain. -3. **Models.** Detectors and meta-agent on `deepseek-v4-flash` through DeepSeek's own OpenAI-compatible endpoint (`https://api.deepseek.com`); `deepseek-v4-pro` only as the second independent judge above suspicion and for classifier disagreements queued for review; embeddings stay on OpenRouter. The chat endpoint and the embedding endpoint are configured separately (today one base-URL variable serves both). The 1M context lets the detector read whole files instead of hotspot excerpts. -4. **Round-zero verifiers.** Snippet templates with an oracle per family, on the existing per-language sandbox recipes: canary row in an in-memory database for injection, canary file for command execution and traversal, canary object for deserialization. Access control gets the obligations checker as static corroboration. Output encoding, SSRF, prototype pollution and configuration report at `suspicion` only until they have an oracle. A family without a verifier never reports above suspicion in a scan but is still scored class-aware on the corpus: the corpus is the verifier for learning, the sandbox is the verifier for reporting. -5. **Obligations lever.** `authorization-obligations` task 5 is retired as a spec task and becomes the access-control family's first evolve round; obligation shapes are that family's memory entries under the split rule. -6. **Order.** Round zero on the web families over vibe-py and agent-vfc, where real-world code, dangerous fruit and a working hunter coincide; the vfc C slice later, as a measured slice only. - -## Evidence - -- `benchmarks/measurements/2026-09-06-vibe-py-holdout-hunter-sonnet5.json`: `ousast pairs --slice vibe-py --split holdout --hunter --hunter-model anthropic/claude-sonnet-5 --json`. -- `benchmarks/measurements/2026-09-06-vibe-py-holdout-hunter-findings.json`: every hunter finding per side with `in_labeled_function` and the scorer's verdict; the class-aware re-score used a keyword classifier written for the measurement only, which is why approach item 2 exists. -- `research.md`: the four research threads with sources. diff --git a/.kiro/specs/learning-harness/design.md b/.kiro/specs/learning-harness/design.md deleted file mode 100644 index 7063b27..0000000 --- a/.kiro/specs/learning-harness/design.md +++ /dev/null @@ -1,651 +0,0 @@ -# Design Document: learning-harness - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Overview - -**Purpose**: Turn OpenUltraSAST from a hand-authored analyzer with a leaking improvement loop into a learning harness: an auto-classifier routes every pair, finding and code region to one of ten verifier-aligned families; one detector configuration per family is evolved one change per round from structured failure facts; a class-aware pair-wise scorer with unscorable denominators and a measured per-family noise floor decides acceptance; only verifier-confirmed claims are reported above suspicion; the split is enforced by construction. - -**Users**: the maintainer runs round zero, evolve rounds and the publish command; operators read scan findings that now carry a family, a detector version and a verifier outcome; reviewers work the classifier disagreement queue. - -**Impact**: the tool hunter becomes the primary detector and the static machinery becomes tools and verifiers. The pair scorer's hunter path, the exporter, leave-one-out, the mechanism lever and the improve journal change behavior; findings, catalogs and the manifest gain additive fields. Gates stay byte-identical. - -### Goals -- One trustworthy number per family per slice, with its denominator, taxonomy version, noise floor and cost. -- A round can only be accepted when it helps its target family on held-out pairs and hurts no other family beyond its measured budget. -- Nothing above `suspicion` without a verifier; verifier code and gold labels outside every editable surface. -- Round zero on the web families over vibe-py and agent-vfc before any learning. - -### Non-Goals -- Browser-engine, interpreter or renderer bugs; crash oracles for C and C++ (the memory-safety family is measured, never gating). -- New static shape families, new hand-written facts, evidence-ladder or disposition changes, merge gates on non-local slices, model fine-tuning. -- A general router framework or a generic agent platform; the classifier is a function and a detector configuration is a directory. - -## Boundary Commitments - -### This Spec Owns -- The family taxonomy file, its loader and its version (1.x). -- Classification of pairs, findings and regions, and the classifier's measurement report and review queue (2.x). -- Class-aware pair scoring: outcomes, directional bias, fixed-FPR view, K-run reliable-change test, negative flips (3.x). -- Unscorable accounting and its reasons; the additive `unscorable` and `family` fields on catalog rows (4.x). -- The single teacher-selection rule every learning path calls, and holdout refusal (5.x). -- Per-family detector configurations, their loader, the family detector runner and finding tags (6.1–6.5); the chat-endpoint resolution independent of embeddings (6.6). -- The verifier registry keyed by family, canary oracles, the second-judge step (7.x). -- Round zero, the noise floor, evolve rounds, acceptance, attribution, the archive, the round journal, cost caps (8.x, 9.x). -- The corpus repairs listed in 10.x and the publish command with committed artifacts (11.x). - -### Out of Boundary -- Pair labels, tiers, splits and `known_limit` semantics beyond the two additive fields (`pair-corpus-honesty`). -- Facts, walker, dominance, obligations checker internals; the mechanism store schema; the rule and policy levers' validators. -- The sandbox runner, snippet safety rules and Docker images; this spec adds an oracle to the verdict, not a runner. -- HarnessX internals; this spec composes `MetaAgent.evolve` and the journal behind `harness_ext`. -- Harvesting new pairs; only re-harvesting existing absence recipes. - -### Allowed Dependencies -- `pairs`, `benchmark`, `semantic.loo`, `semantic.seed`, `semantic.mechanisms`, `improve.evolve`, `improve.validator`, `tool_hunter`, `hunter_tools`, `provider.openrouter`, `regress.*`, `sandbox.*`, `redaction`, `harness_ext`, `config`. -- Optional extras only through `harness_ext` (HarnessX) and `semantic.extra` (tree-sitter, now including TypeScript); every step degrades to a recorded reason without them. -- Direction: `learning.families` → `learning.classify` → `learning.scoring` → `learning.split` → `learning.detectors` and `learning.verifiers` → `learning.rounds` → `learning.publish` and `cli`. `learning.*` imports the packages above; nothing outside `learning` imports `learning.rounds` or `learning.publish`. `pairs` and `benchmark` never import `learning`. - -### Revalidation Triggers -- A change to the family list, a family's CWE set or its verifier (taxonomy version bump). -- A change to the pair outcome vocabulary or the reliable-change test. -- A change to the acceptance rule, the regression-budget definition or the journal schema. -- A change to the chat-endpoint resolution order or the tool-hunter finding shape. -- A change to the sandbox verdict contract (oracle callback) or snippet safety rules. - -## Architecture - -### Existing Architecture Analysis -- `tool_hunter.run_tool_hunter` is a 4-step loop over three curated tools with findings parsed into `StaticFinding` at `suspicion` (`finding_id = tool-hunter::`, tag `tool-hunter`). It refuses to emit findings when no tool was called. It has no temperature, budget or context knobs beyond `max_steps`. -- `pairs._hunter_matches_expected` keys detection on text tokens and `_evaluate_hunter_pair` counts every fixed-side finding as a leak under `fix_policy = silent`; both are replaced on the hunter path only. The overlay and inventory paths and `pair_gate` are untouched. -- `evaluate_profiles`, `evaluate_mechanism_profiles`, `propose_mechanism_edits`, `evaluate_loo` and `build_pair_signals` filter by tier and vendored status but not by split; `export_mechanisms` seeds from every case it is given. -- `OpenRouterChatClient` and `OpenRouterEmbeddingClient` read the same `OPENROUTER_BASE_URL`; `complete_chat` always sends `temperature: 0` and drops `reasoning_content` from replies. DeepSeek requires thinking disabled for temperature to apply, `reasoning_content` replayed on tool-call turns, no `/v1` suffix, `json_object` only, no seed. -- `regress.candidate` derives a verdict from the exit code; snippets are compile-only templates or the model's `proposed_snippet`; `check_snippet_safety` bans sockets, curl and workspace writes, which a canary file in `/scratch` respects. -- `MechanismStore` is an append-only JSONL with tombstones and deterministic ids, the pattern reused for the archive and the journal. -- HarnessX 0.1.0 provides `MetaAgent.evolve(current_config, trajectories_dir, output_dir)` with post-flight checks, `JournalEntrySpec(round, label, hypothesis_id, levers, predicted_affected, prose)`, `build_context` with precision discounted by unpredicted regressions, and `RLTask.task_type` routing. - -### Architecture Pattern & Boundary Map - -```mermaid -graph TB - subgraph Corpus - Catalog[pair catalogs] - Loader[pairs loader with unscorable and family fields] - end - subgraph Learning - Families[families taxonomy] - Classify[classifier three tiers] - Scoring[class aware pair scorer] - Split[teachers and holdout refusal] - Detectors[family detector configs and runner] - Verifiers[verifier registry canary static none] - Rounds[round zero and evolve rounds] - Journal[round journal and archive] - Publish[publish command] - end - subgraph Existing - Hunter[tool hunter loop] - Chat[chat endpoint client] - Regress[regress verdicts and sandbox] - Levers[exporter loo mechanism lever] - Meta[HarnessX MetaAgent optional] - end - Catalog --> Loader --> Scoring - Families --> Classify --> Scoring - Families --> Detectors - Families --> Verifiers - Classify --> Detectors - Split --> Levers - Split --> Rounds - Detectors --> Hunter --> Chat - Verifiers --> Regress - Scoring --> Rounds - Detectors --> Rounds - Verifiers --> Rounds - Rounds --> Journal - Rounds --> Meta - Journal --> Publish - Scoring --> Publish -``` - -**Architecture Integration**: -- Selected pattern: a `learning` package layered left to right (families → classify → scoring → split → detectors/verifiers → rounds → publish), composing existing seams rather than replacing them. -- Domain boundaries: the corpus owns labels and rows; `learning` owns class assignment, scoring, teaching rules, detector configurations, verifier keys and rounds; `regress` owns execution; `harness_ext` owns the HarnessX seam. -- Existing patterns preserved: append-only JSONL with tombstones for the archive and journal; `HarnessRuntime.run_stage` and degradations for every step; `ScriptedChatClient` for offline tests; `FakeSandboxRunner` for verifier tests. -- New components rationale: each is one requirement area with one owner; the classifier and scorer are functions, the detector configuration is a directory, the journal is a file. -- Steering compliance: core `dependencies = []`; recall first under a false-positive ceiling, now per family; gates byte-identical; no model-authored artifact retained without held-out validation. - -### Technology Stack - -| Layer | Choice / Version | Role in Feature | Notes | -|-------|------------------|-----------------|-------| -| Chat endpoint | DeepSeek platform API, OpenAI-compatible, `https://api.deepseek.com` (no `/v1`); models `deepseek-v4-flash` (default), `deepseek-v4-pro` (second judge, review queue) | Detectors, classifier model tier, proposer, judge | Thinking on by default: the client sends `thinking: {"type": "disabled"}` for detectors and the classifier so `temperature` applies; `reasoning_content` replayed on tool-call turns; `response_format json_object` with empty-content retry; `logprobs` requested by the classifier tier only. No seed: determinism is best-effort and absorbed by the noise floor. Pricing (peak, USD/1M): flash 0.44 in / 1.32 out, cache hit 0.014; pro 1.32 / 3.96. Concurrency 2500 (flash). | -| Embedding endpoint | OpenRouter, `openai/text-embedding-3-small`, unchanged | Prove-order retrieval | Reads `OPENROUTER_*` only; DeepSeek has no embeddings endpoint. | -| Evolve loop | HarnessX 0.1.0 (`meta_harness.agent.MetaAgent`, `meta_harness.journal`), optional extra | Proposal generation and attribution context | Behind `harness_ext.require_harnessx`; a scripted proposer stands in without it. | -| Parsing | tree-sitter extra + `tree-sitter-typescript>=0.23` | TypeScript rows become scorable | Optional extra only. | -| Sandbox | existing Docker CLI runner, images `python:3.12-alpine`, `node:22-alpine` | Canary verifiers | Oracle callback added to the verdict; safety rules unchanged. | -| Storage | JSONL under `.openultrasast/learning/` and `benchmarks/measurements/` | Journal, archive, noise floors, published artifacts | Append-only, committed when published. | - -## File Structure Plan - -### Directory Structure -``` -src/openultrasast/learning/ -├── __init__.py # public surface: load_families, classify_*, score_pair_family, teachers, run_baseline, run_learning_round, publish -├── families.py # Family, FamilyTaxonomy, load_families(ruleset/families.toml), version, verifier kind per family -├── classify.py # Classification, classify_pair/finding/region (tiers), measure_classifier, review queue entries -├── scoring.py # FamilyOutcome, score_pair_family, FamilyMetrics, directional_bias, negative_flips, reliable_change, fixed_fpr_view, unscorable_reason -├── split.py # teachers(cases), refuse_if_holdout(candidate_pairs, cases) -> Refusal | None -├── detectors.py # FamilyConfig, load_family_configs, run_family_detector (whole-file context, curated tools), finding tags -├── endpoint.py # ChatEndpoint, resolve_chat_endpoint (DeepSeek first, OpenRouter fallback, scripted), DeepSeekChatClient adapter -├── verifiers.py # Verifier protocol, CanaryVerifier, StaticVerifier, NoVerifier, verifier_for(family), second_judge -├── rounds.py # run_baseline (round zero, noise floor), run_learning_round (propose, minibatch, holdout, sweep, accept/revert), Attribution -├── proposer.py # Proposer protocol, ScriptedProposer, HarnessXProposer (MetaAgent.evolve behind harness_ext), failure facts, rejected buffer -├── journal.py # RoundRecord, LearningJournal (append-only JSONL), Archive (family -> winning config per pair) -└── publish.py # publish(): regenerate numbers from one command, write roadmap table and measurement artifacts -src/openultrasast/ruleset/families.toml # closed, versioned taxonomy: id, cwes, mechanisms, verifier, description -src/openultrasast/learning/configs// # round-zero detector configuration per family (prompt.md, checklist.md, tools.toml, budget.toml, hard_negatives.jsonl, counterexamples.jsonl) -tests/test_learning_families.py, test_learning_classify.py, test_learning_scoring.py, test_learning_split.py, test_learning_detectors.py, test_learning_endpoint.py, test_learning_verifiers.py, test_learning_rounds.py, test_learning_journal.py, test_learning_publish.py, test_fixture_difficulty.py -``` - -### Modified Files -- `src/openultrasast/pairs.py`: `PairCase.unscorable: str | None` computed at load (identical twin via normalized body hash, `known_limit` passthrough); duplicate report across pairs before split; `split_by_repository` helper honoring existing `split` fields and reporting straddles; `_evaluate_hunter_pair` delegates matching and leaks to `learning.scoring` through an injected callable so `pairs` does not import `learning`; `build_pair_signals` gains a `split` filter. -- `src/openultrasast/benchmark.py`: `ExpectedFinding.family: str | None = None` (additive, stored as an opaque string; the Classifier validates loaded labels against the taxonomy so `pairs` and `benchmark` never import `learning`). -- `src/openultrasast/tool_hunter.py`: `run_tool_hunter` gains `system_prompt`, `context_files`, `tools`, `max_chars` and `tags` parameters with today's values as defaults; finding parsing accepts an optional `family` field; the reply loop replays `reasoning_content`. -- `src/openultrasast/hunter_tools.py`: four new tools beside `read_file`, `grep_repo`, `find_refs`: `read_definition(symbol)`, `entry_points(path)`, `flows(path, function)`, `obligations(path, function)`, each clamped to the repository and returning identifiers and line spans only; `HUNTER_TOOLS` schemas extended. -- `src/openultrasast/provider/openrouter.py`: `OpenRouterChatClient` gains `extra_body` (thinking, response_format, logprobs) and retains the `reasoning_content` field on assistant messages; replaying it across tool turns is the hunter loop's job (`tool_hunter`), not the client's. No behavior change for existing callers. -- `src/openultrasast/config.py`: `[models]` gains `chat_base_url`, `chat_api_key_env`, `judge`; `[learning]` section (`families_path`, `configs_dir`, `k_runs = 5`, `round_cost_cap_usd`, `minibatch`, `cutoff_date`); `load_dotenv` unchanged, tests inject endpoints explicitly. -- `src/openultrasast/semantic/seed.py`, `semantic/loo.py`, `improve/evolve.py`: teacher selection through `learning.split.teachers`; refusals recorded; `evaluate_loo` teaches from train only. -- `src/openultrasast/regress/candidate.py`, `regress/verdict.py`: `run_regression` accepts an optional `oracle: Callable[[SandboxResult], bool]`; `verdict_from_result` uses the oracle when given, exit code otherwise. -- `src/openultrasast/cli.py`: `ousast learning classify|score|baseline|round|publish`; `pairs --hunter` uses the family scorer and `--k-runs`; `[models].hunter` resolution through `learning.endpoint`. -- `pyproject.toml`: `tree-sitter-typescript>=0.23` in the semantic extra; `src/openultrasast/semantic/extra.py` maps `typescript`/`tsx`. -- `benchmarks/pairs/vibe-py/catalog.toml` and `recipes.toml`: the five identical twins gain `known_limit = "identical_twin"`; absence rows gain `family` and `mode = "handler_context"` (re-harvest is a maintainer step). -- **Cross-file registration context (amended during task 4.1).** Some frameworks keep the route table out of the handler module: connexion registers by `operationId` in an OpenAPI document, an Express application registers in a router file. A single-file excerpt of such a handler can never contain its registration, so Req 10.2 is unreachable for those rows without a second file. A recipe may therefore name `context = [""]`; `benchmarks/pairs/harvest.py` fetches each document at the same two commits, extracts only the stanza that names the handler (`extract_registration_context`), and writes it under `/context//` with its own provenance header and the same redaction. The catalog carries `[[pair.context]]` rows, `PairCase.context_files` holds `(relpath, vuln, fixed)`, and `_materialize_side` copies them beside the excerpt at their repository-relative paths so the entry-point mapper sees the same layout the project has. `src/openultrasast/mapping.py` gains `_route_manifest_entry_points`, which reads `operationId` and the operation's `security` block out of a YAML or JSON document and attributes the record to the module the operation id points at. Trap pairs whose fixed side is a different, correctly guarded handler carry `fix_function` so the fixed side gets *its* registration rather than the vulnerable side's. -- `benchmarks/pairs/mechanisms.toml`: unchanged; families reference mechanism ids. -- `.kiro/steering/roadmap.md`: rewritten by `publish` from artifacts. - -## System Flows - -### Class-aware scoring of one pair (hunter path) - -```mermaid -sequenceDiagram - participant S as Scorer - participant C as Classifier - participant D as FamilyDetectors - participant V as Verifier - S->>C: classify_pair(case) - C-->>S: families or unknown, tier - loop K runs - S->>D: run on vuln side and fixed side - D-->>S: findings with family tags - end - S->>S: detection = family match inside labeled function - S->>S: silence = no labeled family inside function on fixed side - S->>V: verify claims that a verifier exists for - V-->>S: suspicion, static corroboration, or proven - S-->>S: FamilyOutcome, negative flips, reliable change -``` - -Decisions: unscorable pairs short-circuit before any run; K runs share one classification; verifier outcomes never change detection or silence, they change the reported rung. - -### One evolve round - -```mermaid -flowchart TB - Start[failure facts for target family] --> Propose[proposer drafts one structural change] - Propose --> Refuse{touches holdout or exceeds length cap} - Refuse -->|yes| Reject[reject and record] - Refuse -->|no| Mini[train minibatch of target family] - Mini --> MiniOk{no decrease} - MiniOk -->|no| Reject - MiniOk -->|yes| Hold[target family full holdout] - Hold --> HoldOk{train and holdout non negative and one strictly positive} - HoldOk -->|no| Reject - HoldOk -->|yes| Sweep[every other family holdout] - Sweep --> SweepOk{negative flips within each budget} - SweepOk -->|no| Reject - SweepOk -->|yes| Accept[accept freeze config bump version] - Accept --> Journal[journal hypothesis predicted attribution cost] - Reject --> Journal - Journal --> Buffer[rejected edit buffer updated] -``` - -Decisions: the proposer never sees holdout inputs, outputs or scores; cost is metered per stage and the round is reverted when the cap is crossed at any stage; ties reject. - -## Requirements Traceability - -| Requirement | Summary | Components | Interfaces | Flows | -|---|---|---|---|---| -| 1.1–1.4 | closed taxonomy, verifier per family, version, attributes not keys | Families | `load_families`, `FamilyTaxonomy` | — | -| 2.1–2.5 | three-tier multi-label classifier, measured, review queue, whole corpus | Classifier | `classify_*`, `measure_classifier` | scoring | -| 3.1–3.6 | family-in-function detection and silence, outcomes, bias, FPR, K runs, no text tokens | Scorer | `score_pair_family`, `FamilyMetrics` | scoring | -| 4.1–4.5 | unscorable reasons, never delete, dedup, repository split, denominators shown | Loader (pairs), Scorer, Publish | `PairCase.unscorable`, `unscorable_reason`, `split_by_repository` | — | -| 5.1–5.4 | train-only teachers, refusal, re-measure lever, proposer blind | Split, Levers, Proposer | `teachers`, `refuse_if_holdout` | round | -| 6.1–6.6 | per-family config, frozen, routing plus generalist, whole file, curated tools, independent chat endpoint | Detectors, Endpoint | `FamilyConfig`, `run_family_detector`, `resolve_chat_endpoint` | scoring | -| 7.1–7.6 | suspicion unless verified, canary oracles, static corroboration, none, second judge, sealed surface | Verifiers, Regress | `Verifier`, `verifier_for`, `second_judge`, `run_regression(oracle=)` | scoring | -| 8.1–8.4 | clone, K = 5 noise floor, publish round zero, web families first | Rounds | `run_baseline`, `NoiseFloor` | — | -| 9.1–9.9 | failure facts, one change, journal, staged evaluation, acceptance, attribution, archive, cost cap, retention | Rounds, Proposer, Journal | `run_learning_round`, `Proposer`, `RoundRecord`, `Archive` | round | -| 10.1–10.5 | TypeScript, re-harvest, identical twins, fixture difficulty, post-cutoff slice | Extra, Corpus, Tests | `semantic.extra`, catalog fields, `test_fixture_difficulty` | — | -| 11.1–11.5 | one command, artifacts committed, trajectories persisted, degrade with reason, gates identical | Publish, Journal, Runtime | `publish`, `LearningJournal` | — | - -## Components and Interfaces - -| Component | Layer | Intent | Req | Key Dependencies | Contracts | -|---|---|---|---|---|---| -| Families | data | closed taxonomy with verifier per family | 1.x | ruleset file (P0) | Service | -| Classifier | logic | assign families with tier, measure itself | 2.x | Families (P0), Endpoint (P1) | Service, State | -| Scorer | logic | class-aware pair outcomes and statistics | 3.x, 4.x | Families (P0); findings from Detectors and rungs from Verifiers are inputs, not imports (P1) | Service | -| Split | logic | the one teacher rule and holdout refusal | 5.x | pairs (P0) | Service | -| Detectors | logic | per-family configuration and runner | 6.1–6.5 | tool_hunter (P0), Endpoint (P0), Classifier (P0) | Service, State | -| Endpoint | integration | independent chat endpoint resolution and DeepSeek adapter | 6.6 | provider.openrouter (P0), config (P0) | Service | -| Verifiers | integration | family-keyed oracles and the second judge | 7.x | regress, sandbox (P0), obligations (P1) | Service | -| Rounds | orchestration | round zero, noise floor, evolve rounds, acceptance | 8.x, 9.x | Scorer, Detectors, Proposer, Journal (P0) | Batch, State | -| Proposer | integration | structural change proposals from failure facts | 9.1–9.2 | harness_ext (P1), Endpoint (P1) | Service | -| Journal | data | round records, attribution, archive | 9.3, 9.6, 9.7, 11.3 | JSONL (P0) | State | -| Publish | orchestration | numbers from one command, artifacts, roadmap | 11.x | Scorer, Journal (P0) | Batch | - -### Data layer - -#### Families - -| Field | Detail | -|---|---| -| Intent | Load and version the closed family taxonomy with its CWE set, mechanism set and verifier kind | -| Requirements | 1.1, 1.2, 1.3, 1.4 | - -**Responsibilities & Constraints** -- Exactly ten families as decided; ids are stable identifiers; the file carries `version`. -- Verifier kind per family is one of `canary`, `static`, `none`; memory safety is `none`. -- No component may route or score on a CWE or mechanism id; they are looked up only through a family. - -**Dependencies**: Inbound: Classifier, Scorer, Detectors, Verifiers (P0). External: `ruleset/families.toml` (P0). - -**Contracts**: Service [x] - -```python -@dataclass(frozen=True) -class Family: - id: str # e.g. "injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "prototype", "config_secrets", "memory", "unknown" - cwes: frozenset[str] - mechanisms: frozenset[str] - verifier: Literal["canary", "static", "none"] - description: str - -@dataclass(frozen=True) -class FamilyTaxonomy: - version: str - families: tuple[Family, ...] - def by_id(self, family_id: str) -> Family: ... - def family_of_cwe(self, cwe: str) -> Family | None: ... - def family_of_mechanism(self, mechanism: str) -> Family | None: ... - def related(self, a: str, b: str) -> Literal["same", "parent_child", "lateral", "fabricated"]: ... - -def load_families(path: Path | None = None) -> FamilyTaxonomy # raises FamiliesError naming the field on any deviation -``` -- Invariant: `unknown` is always present and never has a verifier. - -#### Journal - -| Field | Detail | -|---|---| -| Intent | Append-only record of rounds with attribution, and the archive of which configuration wins which pair | -| Requirements | 9.3, 9.6, 9.7, 11.3 | - -**Contracts**: State [x] - -```python -@dataclass(frozen=True) -class RoundRecord: - round: int - family: str - hypothesis: str - levers: tuple[str, ...] # "tool" | "checklist" | "counterexample" | "memory" | "prompt" - predicted_affected: tuple[str, ...] - predicted_at_risk: tuple[str, ...] - outcome: Literal["accepted", "rejected", "reverted_cost", "refused_holdout"] - reason: str - target_train_delta: float - target_holdout_delta: float - sweep: dict[str, int] # family -> negative flips - attribution: Attribution # flipped_predicted, flipped_unpredicted, precision - cost_usd: float - taxonomy_version: str - config_version: str - -class LearningJournal: # .openultrasast/learning/journal.jsonl - def append(self, record: RoundRecord) -> None: ... - def rounds(self) -> tuple[RoundRecord, ...]: ... - def rejected_buffer(self, family: str) -> tuple[str, ...]: ... # hypotheses rejected for this family - -class Archive: # .openultrasast/learning/archive.jsonl - def record(self, family: str, config_version: str, pair: str, outcome: FamilyOutcome) -> None: ... - def winners(self, family: str) -> dict[str, str]: ... # pair -> config version that scores it best -``` -- Persistence: JSONL, one record per line, never rewritten; the HarnessX journal is mirrored when the extra is present, never the source of truth. - -### Logic layer - -#### Classifier - -| Field | Detail | -|---|---| -| Intent | Assign zero or more families with the deciding tier; measure itself; queue disagreements | -| Requirements | 2.1, 2.2, 2.3, 2.4, 2.5 | - -**Responsibilities & Constraints** -- Tier 1 (deterministic): rule id, sink family from facts, obligation kind, mechanism id, labeled CWE; any hit decides without a model. -- Tier 2 (model): `json_object` question with the taxonomy definitions, `thinking` disabled, `logprobs` requested; averaged log-probability under a threshold yields `unknown`. -- Tier 3: `unknown`. -- Never writes a label; disagreements go to `.openultrasast/learning/review-queue.jsonl`. -- Validates every loaded catalog `family` label against the taxonomy at classification time and rejects a fabricated id by name; the corpus loader stores the label as an opaque string. - -**Dependencies**: Inbound: Scorer, Detectors (P0). Outbound: Families (P0), Endpoint (P1, degrades to tier 1 + unknown without a client). - -**Contracts**: Service [x] State [x] - -```python -@dataclass(frozen=True) -class Classification: - families: tuple[str, ...] # () means unknown - tier: Literal["static", "model", "none"] - confidence: float | None - -def classify_pair(case: PairCase, taxonomy: FamilyTaxonomy, *, client: ChatClient | None) -> Classification -def classify_finding(finding: StaticFinding, taxonomy: FamilyTaxonomy, *, client: ChatClient | None) -> Classification -def classify_region(path: str, text: str, function: str | None, taxonomy: FamilyTaxonomy, *, facts, client: ChatClient | None) -> Classification - -@dataclass(frozen=True) -class ClassifierReport: - taxonomy_version: str - agreement: float # hierarchical credit vs maintainer labels on reviewed pairs - confusion: dict[str, dict[str, int]] - random_router: float - oracle_router: float - per_tier: dict[str, int] - unknown: int - queued: int - -def measure_classifier(cases: Sequence[PairCase], taxonomy: FamilyTaxonomy, *, client: ChatClient | None) -> ClassifierReport -``` -- Postcondition: every reviewed pair whose label disagrees with the classification appears once in the review queue with both answers and the tier. - -#### Scorer - -| Field | Detail | -|---|---| -| Intent | Score one pair per family from detector findings, aggregate per family and slice, decide reliable change | -| Requirements | 3.1–3.6, 4.1, 4.5 | - -**Responsibilities & Constraints** -- Detection: a finding whose `family:` tag relates `same` or `parent_child` to the labeled family and whose line lies in the labeled function; `fabricated` scores the maximum penalty; text is never inspected. -- Silence: no `same`/`parent_child` finding of the labeled family inside the labeled function on the fixed side; other findings are counted in `other_findings`, never as leaks. -- K runs per pair at every evaluation (baseline K = 5, rounds K ≥ 3 as 3.5 requires); reliable change against the last blessed round via a paired test on per-pair outcomes; negative flips per family. - -**Contracts**: Service [x] - -```python -FamilyOutcome = Literal["pair_correct", "both_flagged", "both_silent", "reversed", "unscorable"] - -@dataclass(frozen=True) -class PairFamilyScore: - pair: str - family: str - runs: tuple[FamilyOutcome, ...] # length K - outcome: FamilyOutcome # majority of runs - other_findings_vuln: int - other_findings_fixed: int - rung: Literal["suspicion", "static_corroboration", "proven"] - unscorable_reason: str | None - -@dataclass(frozen=True) -class FamilyMetrics: - family: str - scorable: int - unscorable: dict[str, int] # reason -> count - outcomes: dict[FamilyOutcome, int] - recall: float - silence: float - youden: float - directional_bias: float # (both_flagged - both_silent) / scorable - fixed_fpr_recall: float # recall at the family's FPR ceiling - negative_flips: int # vs last blessed round - reliable_change: bool - taxonomy_version: str - -def score_pair_family(case: PairCase, family: str, vuln_runs: Sequence[Sequence[StaticFinding]], fixed_runs: Sequence[Sequence[StaticFinding]], *, ranges, taxonomy) -> PairFamilyScore -def aggregate(scores: Sequence[PairFamilyScore], *, baseline: Sequence[PairFamilyScore] | None) -> dict[str, FamilyMetrics] -def unscorable_reason(case: PairCase, *, parse_ok: bool, ranges) -> str | None # identical_twin | unsupported_language | unresolved_label | missing_context | known_limit: -``` - -#### Split - -| Field | Detail | -|---|---| -| Intent | The one rule that selects teachers, and the refusal for anything touching holdout | -| Requirements | 5.1, 5.2, 5.3, 5.4 | - -**Contracts**: Service [x] - -```python -def teachers(cases: Sequence[PairCase]) -> tuple[PairCase, ...] # vendored, gating tier, split == "train", scorable -def refuse_if_holdout(taught_by: Iterable[str], cases: Sequence[PairCase]) -> Refusal | None # Refusal(pairs=..., reason="holdout_pair") -``` -- `export_mechanisms`, `evaluate_loo`, `evaluate_profiles`, `evaluate_mechanism_profiles`, `propose_mechanism_edits` and `build_pair_signals` obtain teachers only through `teachers`; refusals are recorded as degradations `{"stage": "learning", "reason": "holdout_pair_refused", "pairs": [...]}`. - -#### Detectors - -| Field | Detail | -|---|---| -| Intent | Load frozen per-family configurations and run the family detector with whole-file context and curated tools | -| Requirements | 6.1, 6.2, 6.3, 6.4, 6.5 | - -**Responsibilities & Constraints** -- A configuration is a directory `learning/configs//` with `prompt.md` (length cap enforced at load), `checklist.md`, `tools.toml` (subset of `read_file`, `grep_repo`, `find_refs`, `read_definition`, `entry_points`, `flows`, `obligations`), `budget.toml` (`max_steps`, `max_cost_usd`, `max_chars`), `hard_negatives.jsonl`, `counterexamples.jsonl`, `version`. -- Round zero writes every family's directory from the current hunter prompt; a round rewrites only its target family and bumps that `version`. -- The runner builds the user message from the whole file plus resolved definitions, runs `run_tool_hunter` with the family's tools and prompt, and tags findings `family:`, `detector:@`; the generalist is the `unknown` family's configuration. - -**Contracts**: Service [x] State [x] - -```python -@dataclass(frozen=True) -class FamilyConfig: - family: str - version: str - prompt: str - checklist: str - tools: tuple[str, ...] - max_steps: int - max_cost_usd: float - max_chars: int - hard_negatives: tuple[dict[str, object], ...] - counterexamples: tuple[dict[str, object], ...] - -def load_family_configs(configs_dir: Path, taxonomy: FamilyTaxonomy) -> dict[str, FamilyConfig] # raises ConfigError on cap or tool violations -def run_family_detector(root: Path, region: Region, config: FamilyConfig, *, client: ChatClient, model: str) -> list[StaticFinding] -``` -- Precondition: `region.families` from the classifier; postcondition: every finding carries exactly one `family:` tag and one `detector:` tag. - -#### Endpoint - -| Field | Detail | -|---|---| -| Intent | Resolve the chat endpoint independently of the embedding endpoint and adapt DeepSeek quirks | -| Requirements | 6.6 | - -**Responsibilities & Constraints** -- Resolution order: explicit injection (tests) → `OPENULTRASAST_HUNTER_CLIENT` scripted modes → `DEEPSEEK_API_KEY` with `DEEPSEEK_BASE_URL` (default `https://api.deepseek.com`) → `OPENROUTER_API_KEY` with `OPENROUTER_BASE_URL`. Embeddings keep reading `OPENROUTER_*` only. -- `[models]`: `hunter` (default `deepseek-v4.1-flash`), `judge` (default `deepseek-v4.1-flash`), `chat_base_url`, `chat_api_key_env`. - V4 Pro was discontinued on 2026-09-10 and its requests route to V4.1 Flash from 2026-09-14, billed at Flash's - price. Detector and judge are now the SAME model, so any claim that the judge is a stronger second opinion is - a claim about the prompt and the evidence put to it, not about model class. -- Adapter: `thinking` disabled and `temperature 0` for detectors and classifier; `reasoning_content` replayed; `response_format json_object` with one retry on empty content; `logprobs` on request; usage fields normalized (`prompt_cache_hit_tokens`) into cost accounting. - -**Contracts**: Service [x] - -```python -@dataclass(frozen=True) -class ChatEndpoint: - base_url: str - api_key: str - provider: Literal["deepseek", "openrouter", "scripted"] - thinking: bool - price_in_per_m: float - price_out_per_m: float - -def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None = None) -> tuple[ChatClient, ChatEndpoint] | None -class DeepSeekChatClient: # implements ChatClient; wraps OpenRouterChatClient with extra_body and reasoning_content replay - def complete(self, *, model, messages, tools, timeout_seconds=60, json_object=False, logprobs=False) -> ChatResponse -``` - -#### Verifiers - -| Field | Detail | -|---|---| -| Intent | Confirm a claim through the family's oracle, or refuse to raise it | -| Requirements | 7.1–7.6 | - -**Responsibilities & Constraints** -- `CanaryVerifier` (injection, path, deserialization, and command execution inside injection): writes a snippet from a family template that imports the target function, plants a canary (row in an in-memory database, file under `/scratch`, object with a marker method) and prints `CANARY:` only when the vulnerability fires; the oracle is the token in stdout, never the exit code. The snippet passes `check_snippet_safety` unchanged. -- `StaticVerifier` (access control): the obligations checker's finding on the same function raises to `static_corroboration`. -- `NoVerifier`: stays at `suspicion`. -- `second_judge`: a `deepseek-v4.1-flash` call with the snippet, the oracle output and the claim, answering `json_object` `{"confirms": bool, "reason": str}`; a proven claim is published only when it confirms. -- Templates, oracle code and gold labels live under `learning/verifiers/` and `benchmarks/`, both excluded from every proposer write root. - -**Contracts**: Service [x] - -```python -class Verifier(Protocol): - kind: Literal["canary", "static", "none"] - def verify(self, claim: StaticFinding, *, root: Path, language: str, runner: SandboxRunner, limits: SandboxConfig) -> Verification - -@dataclass(frozen=True) -class Verification: - rung: Literal["suspicion", "static_corroboration", "proven"] - oracle_output: str - reason: str - -def verifier_for(family: Family) -> Verifier -def second_judge(verification: Verification, claim: StaticFinding, *, client: ChatClient, model: str) -> bool -``` -- `regress.run_regression(..., oracle=Callable[[SandboxResult], bool] | None)`: when given, `TRIGGERABLE` iff the oracle returns true; exit-code behavior unchanged otherwise. - -### Orchestration layer - -#### Rounds - -| Field | Detail | -|---|---| -| Intent | Round zero with noise floors; evolve rounds with staged evaluation, acceptance, attribution and revert | -| Requirements | 8.1–8.4, 9.1–9.9 | - -**Responsibilities & Constraints** -- `run_baseline`: clone the current hunter prompt into every family directory, run K = 5 over scorable vendored pairs of the selected slices, compute per-family negative-flip rate between runs as `NoiseFloor`, persist `noise-floors.json`, publish round zero. Artifacts and floors are keyed by detector model (`baseline//`), so a second model is baselined by changing `[models].hunter` and re-running; `publish` renders a model comparison table from every baseline present. This is the maintainer's reconsideration path if the default model underperforms; configurations are never touched by a model swap. -- `run_learning_round`: failure facts for the target family → proposer → refusal checks (holdout, length cap, allowed levers) → minibatch → holdout → sweep → accept (freeze, bump version, archive) or revert (restore the directory snapshot) → journal → rejected buffer. -- Acceptance: `train_delta >= 0 and holdout_delta >= 0 and (train_delta > 0 or holdout_delta > 0) and all(sweep[f] <= floor[f])`; ties reject. -- Cost metered per stage from usage; crossing `round_cost_cap_usd` at any stage reverts and records `reverted_cost`. -- The proposer receives no holdout material; scoring of holdout happens after the proposal is frozen. - -**Contracts**: Batch [x] State [x] - -```python -@dataclass(frozen=True) -class NoiseFloor: - family: str - k_runs: int - negative_flip_rate: float - budget_flips: int # ceil(rate * scorable pairs) - -def run_baseline(cases, *, slices, taxonomy, configs_dir, client, model, k_runs=5, out_dir) -> BaselineReport -def run_learning_round(cases, *, family, taxonomy, configs_dir, proposer, client, model, journal, archive, floors, cost_cap_usd, minibatch) -> RoundRecord -``` -- Idempotency: a round directory is written under `.openultrasast/learning/rounds//` with the snapshot, proposal, scores and trajectories; rerunning a round number refuses. -- Every stage scores with K runs as the scorer requires; the noise floor is the tolerance for the cross-family sweep, not a substitute for repeated runs. - -#### Proposer - -| Field | Detail | -|---|---| -| Intent | Draft one structural change for one family from structured failure facts | -| Requirements | 9.1, 9.2, 5.4 | - -**Contracts**: Service [x] - -```python -@dataclass(frozen=True) -class FailureFacts: - family: str - misses: tuple[MissFact, ...] # pair, function, sink_or_check, line, verifier_outcome, runs - leaks: tuple[LeakFact, ...] - rejected: tuple[str, ...] # hypotheses from the buffer - config: FamilyConfig - -@dataclass(frozen=True) -class Proposal: - hypothesis: str - lever: Literal["tool", "checklist", "counterexample", "memory", "prompt"] - change: dict[str, object] # file-level edit inside the family directory - predicted_affected: tuple[str, ...] - predicted_at_risk: tuple[str, ...] - -class Proposer(Protocol): - def propose(self, facts: FailureFacts) -> Proposal | None - -class ScriptedProposer: ... # tests and offline runs -class HarnessXProposer: ... # MetaAgent.evolve with allowed_write_roots = the family directory only; degrades to None with a recorded reason without the extra -``` - -#### Publish - -| Field | Detail | -|---|---| -| Intent | Regenerate every published number from one command and commit its artifacts | -| Requirements | 11.1, 11.2, 4.5 | - -**Contracts**: Batch [x] - -```python -def publish(cases, *, taxonomy, journal, floors, out_dir=Path("benchmarks/measurements"), roadmap=Path(".kiro/steering/roadmap.md")) -> PublishReport -``` -- Writes `--.json` artifacts, a markdown table with scorable/unscorable counts, taxonomy version, noise floor and cost per point beside every number, and replaces the roadmap's measured section between markers. - -## Data Models - -### Domain Model -- Aggregates: `FamilyTaxonomy` (versioned, immutable per version); `FamilyConfig` (per family, versioned, frozen between rounds); `RoundRecord` (append-only); `PairCase` (owned by the corpus, gains two additive fields). -- Invariants: a finding carries exactly one `family:` tag; a pair has at most one `unscorable` reason; a round targets exactly one family; the archive maps a pair to one winning configuration per family. - -### Data Contracts -- Catalog row additions: `[[pair.expected]] family = ""` (opaque at load, validated by the Classifier), `known_limit = "identical_twin"` for the five twins, `fix_date = "YYYY-MM-DD"` populated by the catalog generator from recipe metadata where available (rows without it are counted as undated in the post-cutoff slice); `unscorable` is computed, never stored. -- Manifest block `learning = {taxonomy_version, families: {id: FamilyMetrics}, classifier: ClassifierReport, round: n | null}`; findings carry tags `family:`, `detector:`, `verifier:`. -- Journal and archive JSONL as above; `noise-floors.json`; `review-queue.jsonl` entries `{pair, label_family, classified, tier, confidence}`. - -## Error Handling - -- Taxonomy or configuration deviations fail fast at load with the field named (`FamiliesError`, `ConfigError`). -- Missing extras degrade with a recorded reason: no HarnessX → `ScriptedProposer` or `proposer_unavailable`; no TypeScript grammar → rows `unsupported_language`; no sandbox → `NoVerifier` for canary families with `sandbox_unavailable`. -- Endpoint errors: transient retries as today; empty `json_object` content retried once then recorded; a round whose endpoint fails mid-stage is reverted with `endpoint_failed`. -- Holdout refusal, cost overrun and safety rejection are outcomes, not exceptions, and always land in the journal. -- Monitoring: every stage runs under `HarnessRuntime.run_stage`; cost per stage and per round is written to the round directory. - -## Testing Strategy - -- Unit: taxonomy loader rejects an eleventh family and a missing verifier kind (1.1, 1.2); `related` returns `parent_child` for a CWE child and `fabricated` for an unknown id (3.1); classifier tier 1 decides from a rule id without a client and tier 2 falls to `unknown` under the threshold with a scripted client (2.1, 2.2); scorer counts detection only inside the labeled function and never from text (3.1, 3.6); silence ignores other-family findings (3.2); directional bias and negative flips on synthetic run sets (3.3, 3.5); `unscorable_reason` on an identical twin and an unsupported language (4.1); `teachers` excludes holdout and `refuse_if_holdout` names the pair (5.1, 5.2); configuration loader rejects a prompt over the cap and a tool outside the allowed set (6.1, 6.5); endpoint resolution prefers DeepSeek over OpenRouter and never touches the embedding variables (6.6); canary snippet passes safety and the oracle keys on the token (7.2); acceptance rule rejects ties and budget breaches (9.5); journal round-trips a record and the rejected buffer (9.3, 9.6). -- Integration: `pairs --hunter` with a scripted client on the shared handler fixture yields family outcomes and the manifest block (3.x, 6.3); `export_mechanisms` and `evaluate_loo` teach from train only and record refusals (5.1, 5.3); `run_baseline` with a scripted client writes noise floors and round-zero artifacts (8.x); `run_learning_round` with `ScriptedProposer` accepts a helpful change, rejects a tie, reverts on cost and journals attribution (9.x); `run_regression(oracle=)` returns `TRIGGERABLE` on the token with `FakeSandboxRunner` (7.2). -- Corpus: TypeScript rows parse under the extra (10.1); the five twins load as `known_limit:identical_twin` and are excluded from denominators (10.3, 4.1); `test_fixture_difficulty` fails on a fixture easier than its named rows (10.4); post-cutoff slice reported separately (10.5). -- Gates: `gate`, `map_gate`, `pair_gate` stdout byte-identical to the pre-feature commit; extra-free suite green with semantic and harnessx tests skipping (11.5). -- Live (maintainer, not CI): round zero on vibe-py and agent-vfc web families through DeepSeek, artifacts committed under `benchmarks/measurements/`. - -## Security Considerations -- Every prompt and every persisted trajectory passes `redact_secrets`; verifier templates, oracle code and gold labels are outside the proposer's write roots; snippets keep today's safety bans; the sandbox stays network-off and workspace read-only; the second judge sees the snippet and oracle output, never the labels. - -## Performance & Cost -- Round zero: K = 5 over ~250 vendored pairs, both sides, whole-file context ≈ 25M input tokens; at DeepSeek flash peak pricing under $15, off-peak half. A round: minibatch + holdout + sweep ≈ one baseline pass; `round_cost_cap_usd` default 10. Concurrency bounded by the runtime's existing budget controls. - -## Migration Strategy -- Phase 1 (offline, no model): taxonomy, classifier tier 1, scorer, split, unscorable rows, endpoint resolution, TypeScript grammar, identical twins, fixture-difficulty check; existing numbers re-published with denominators. -- Phase 2 (live): round zero on web families; noise floors; classifier tier 2 measured; review queue worked. -- Phase 3: verifiers for the four canary families and the static verifier; second judge. -- Phase 4: evolve rounds with `ScriptedProposer` first, then `HarnessXProposer`; obligations lever as the access-control family's first round. -- Rollback: every round is a directory snapshot; `publish` regenerates from artifacts, so a bad round is reverted by restoring the family directory and re-publishing. diff --git a/.kiro/specs/learning-harness/requirements.md b/.kiro/specs/learning-harness/requirements.md deleted file mode 100644 index 01f1a09..0000000 --- a/.kiro/specs/learning-harness/requirements.md +++ /dev/null @@ -1,156 +0,0 @@ -# Requirements Document - -> **Status 2026-10-02:** retired, superseded by `model-grounded-detection` (spec.json `superseded_by`). 30 of 33 tasks were done; the open three will not be. Kept as decision history; nothing in src, tests or CI points here. - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Introduction - -OpenUltraSAST's detection numbers cannot be trusted and cannot be improved. The static machinery is treated as the detector while the model with tools already outperforms it (12/14 versus 5/14 pair-correct on the vibe-py holdout once scoring is keyed on class and function), the pair scorer keys detection on text tokens rather than on a vulnerability class, the improve loop learns from its own holdout pairs, and no change is attributed to the class it targets, so an improvement in one class silently degrades another. This feature turns the tool into a learning harness: an auto-classifier assigns every labeled pair, finding and code region to one of ten verifier-aligned families; one detector configuration per family is evolved round by round from structured failure facts; a class-aware, pair-wise scorer with unscorable denominators and a measured per-family noise floor decides acceptance; only verifier-confirmed claims are reported above suspicion; and the train/holdout split is enforced by construction. Context, measurements and sources: `brief.md`, `research.md`, `benchmarks/measurements/2026-09-06-*`. - -## Boundary Context - -- In scope: the class taxonomy and its classifier; the class-aware pair scorer and unscorable-row accounting; split enforcement for every learning path; per-family detector configurations and their routing; class-keyed verifiers and reporting rungs; the round-based evolve loop with attribution; the corpus repairs the loop needs; publishing every number with its denominator and noise floor. -- Out of scope: finding bugs in JavaScript interpreters, WebGL renderers or browser engines; memory-safety fuzzing or crash oracles for C and C++ (the vfc slice stays a measured, non-gating slice under one coarse family); new hand-written static shape families; changes to the evidence ladder or dispositions; merge gates on non-local slices; vendoring new pairs beyond re-harvesting existing recipes; fine-tuning model weights. -- Adjacent expectations: the pair corpus (`pair-corpus-honesty`) keeps owning labels, tiers, splits and `known_limit`; `corpus-seeded-mechanisms` keeps its exporter, leave-one-out and lever, which this feature subordinates to the split and the class attribution; `authorization-obligations` keeps its checker, whose corpus tasks 4.3 and 6.1 move here and whose lever task 5 becomes the access-control family's first round; the local detection, map and pair gates stay byte-identical; HarnessX and tree-sitter grammars remain optional extras and every step that needs them degrades to a recorded reason without them. - -## Requirements - -### Requirement 1: A closed, versioned class taxonomy aligned to verifiers - -**Objective:** As a maintainer, I want every unit of detection work to belong to one of a small set of named families, so that detection, silence, leaks and improvements have a definite meaning. - -#### Acceptance Criteria - -1. The Learning Harness shall define a closed, versioned taxonomy of exactly these families: injection into a query or command; file path from user input; deserialization of untrusted data; access control; output encoding; untrusted destination (server-side request forgery and open redirect); prototype pollution and unsafe object merge; permissive configuration and secrets; memory safety; unknown. -2. The Learning Harness shall record for every family the CWE identifiers it covers, the mechanism ids it covers, and the verifier that can confirm a claim in that family or the statement that none exists. -3. When the taxonomy changes, the Learning Harness shall bump its version and every published number shall name the taxonomy version it was scored under. -4. The Learning Harness shall treat CWE identifiers and mechanism ids as attributes of a family and shall never use them as the key that routes work or scores a pair. - -### Requirement 2: An auto-classifier that is measured on its own - -**Objective:** As a maintainer, I want a classifier that assigns families to pairs, findings and code regions and reports its own accuracy, so that routing errors are visible instead of hidden inside detector numbers. - -#### Acceptance Criteria - -1. When given a labeled pair, a finding or a code region, the Learning Harness shall return zero or more families plus, when nothing is admitted, the `unknown` family, and shall record which tier decided each answer: deterministic static signals, a model judgment, or none. -2. The Learning Harness shall attempt deterministic classification from static signals before consulting a model, and shall consult a model only for what static signals cannot place. -3. When the classifier is evaluated, the Learning Harness shall report agreement with maintainer labels on reviewed pairs with partial credit for a parent–child relation and no credit for a lateral or fabricated family, a confusion matrix per family, and its accuracy against both a random router and an oracle router. -4. When the classifier disagrees with a maintainer label, the Learning Harness shall queue the disagreement for review and shall not change the label. -5. The Learning Harness shall classify every labeled row in the corpus, including rows whose current class is free text or `other`, and shall report the count classified per family and the count left `unknown`. -6. While no family in the taxonomy declares a parent, the Learning Harness shall state beside every agreement figure that hierarchical partial credit could not apply. - -### Requirement 3: A class-aware, pair-wise scorer - -**Objective:** As a maintainer, I want pairs scored on whether the labeled family was found in the labeled function, so that a detector is credited for the bug it was asked about and nothing else. - -#### Acceptance Criteria - -1. When scoring the vulnerable side, the Learning Harness shall count a detection only for a finding inside the labeled function whose family matches the labeled family, with partial credit for a parent–child relation and the maximum penalty for a fabricated family. -2. When scoring the fixed side, the Learning Harness shall count silence only when no finding of the labeled family lies inside the labeled function, and shall report findings of other families or outside the function separately, never as leaks. -3. The Learning Harness shall report every pair as exactly one of pair-correct, both-sides-flagged, both-sides-silent, or reversed, and shall report per family and per slice the counts of each outcome, recall, silence, Youden and a directional-bias index. -4. The Learning Harness shall report a fixed-false-positive-rate view per family beside Youden. -5. When a detector is evaluated, the Learning Harness shall run it at least three times per pair and shall report a change against the last blessed round only when a reliable-change test passes, together with the negative-flip count per family. -6. The Learning Harness shall never count a text token, a CWE string or a sink name in a finding's text as evidence of detection. -7. While no family in the taxonomy declares a parent, the Learning Harness shall state beside every published family figure that hierarchical partial credit could not apply. - -### Requirement 4: Unscorable rows and corpus hygiene - -**Objective:** As a maintainer, I want rows that cannot be scored removed from every denominator with a stated reason, so that no number carries a permanent loss that no detector could avoid. - -#### Acceptance Criteria - -1. The Learning Harness shall mark a pair unscorable with one of these reasons: identical twin, unsupported language, unresolved function label, missing registration context, or a maintainer-declared known limit; and shall exclude it from recall, silence, Youden and negative-flip denominators while still listing it. -2. The Learning Harness shall never delete a catalog row; an unscorable row shall keep its reason in the catalog. -3. When pairs are loaded, the Learning Harness shall detect identical twins by comparing normalized excerpt bodies with headers stripped, and shall report duplicates across pairs before any split is applied. -4. The Learning Harness shall assign train and holdout membership by repository and in date order, so that no two pairs from one repository straddle the split. -5. When a published number is shown, the Learning Harness shall show the scorable count, the unscorable count by reason, and the taxonomy version beside it. - -### Requirement 5: The split is enforced by construction - -**Objective:** As a maintainer, I want it impossible for any learning path to see a holdout pair, so that a holdout number means what it says. - -#### Acceptance Criteria - -1. When any exporter, lever or evolve round seeds, proposes or validates from pairs, the Learning Harness shall use only train-split pairs as teachers. -2. If a candidate change was taught by, or touches, any holdout pair, then the Learning Harness shall refuse it and record the refusal with the pair names. -3. The Learning Harness shall re-measure the existing mechanism lever under this rule and shall publish the corrected holdout number in place of the earlier one. -4. While an evolve round proposes changes, the Learning Harness shall not expose holdout inputs, outputs or scores to the proposer. - -### Requirement 6: One detector configuration per family - -**Objective:** As a maintainer, I want each family to have its own detector configuration that other families' changes cannot alter, so that improvements do not interfere. - -#### Acceptance Criteria - -1. The Learning Harness shall hold one detector configuration per family consisting of a prompt under a stated length cap, a curated tool set, a step budget, a cost budget, an evidence checklist and a set of hard negatives and counterexamples. -2. While a round evolves one family, the Learning Harness shall keep every other family's configuration unchanged. -3. When a code region is classified, the Learning Harness shall run the detector of every admitted family plus a generalist detector, and shall record on each finding the family, the detector configuration version and the verifier outcome. -4. The Learning Harness shall give a detector the whole file of a region and the files its curated tools resolve instead of limiting input to an excerpt around a hotspot. -5. The Learning Harness shall expose to a detector only curated tools that read, search and resolve symbols within the scanned repository, and no general shell. -6. The Learning Harness shall run detectors, the classifier's model tier and the evolve proposer on a configurable chat endpoint that is independent of the embedding endpoint, with a cheap model as the default and a stronger model reserved for the second judge and for review-queue disagreements. - -### Requirement 7: Only a verifier raises a claim above suspicion - -**Objective:** As an operator, I want to know that anything reported above suspicion was confirmed by something other than the model, so that reports stay honest. - -#### Acceptance Criteria - -1. The Learning Harness shall report a detector claim at `suspicion` unless a verifier keyed to the claim's family confirms it. -2. Where a family has a sandbox oracle, the Learning Harness shall confirm a claim by executing a family-specific snippet whose oracle is a canary observable only through the vulnerability (a canary row for injection, a canary file for command execution and traversal, a canary object for deserialization). -3. Where a family has only static corroboration (access control through the obligations checker), the Learning Harness shall raise a claim no higher than static corroboration. -4. Where a family has no verifier, the Learning Harness shall report at `suspicion` only and shall still score that family class-aware on the corpus. -5. When a claim is reported as proven, the Learning Harness shall obtain a second, independent judgment before publishing it as proven. -6. The Learning Harness shall keep verifier code, oracles and gold labels outside every surface an evolve round can edit. - -### Requirement 8: Round zero establishes the baseline and the noise floor - -**Objective:** As a maintainer, I want a controlled baseline and a measured noise floor per family before any learning, so that later gains and regressions are distinguishable from noise. - -#### Acceptance Criteria - -1. When round zero runs, the Learning Harness shall clone the current hunter into every family's configuration unchanged. -2. When round zero runs, the Learning Harness shall run each family's detector five times over the scorable vendored pairs at deterministic settings and shall record the per-family negative-flip rate between runs as that family's regression budget. -3. The Learning Harness shall publish round zero's per-family outcomes, noise floors and cost per pair before the first evolve round. -4. The Learning Harness shall run round zero first on the web families over the vibe-py and agent-vfc slices; the memory-safety family over the vfc slice is measured but never gates a round. - -### Requirement 9: An evolve round improves one family under attribution - -**Objective:** As a maintainer, I want each round to change one thing for one family and to be accepted only when the change helps that family without hurting any other, so that the system cannot degrade in ways nobody predicted. - -#### Acceptance Criteria - -1. When a round starts, the Learning Harness shall give the proposer structured failure facts for the target family (which pair, which function, which sink or check, which line, which verifier outcome) and the buffer of previously rejected proposals, and shall not give it free-form self-critique or holdout material. -2. The Learning Harness shall accept at most one change to one family per round, and the change shall be a tool, a checklist, a counterexample, a memory entry, or a prompt edit within the family's length cap. -3. When a round records its proposal, the Learning Harness shall journal a hypothesis, the levers changed, the family predicted to improve, and the families predicted to be at risk. -4. When a round is evaluated, the Learning Harness shall score the target family on a train minibatch, then on the target family's full holdout, then sweep every other family's holdout. -5. If the target family's train score or holdout score decreases, or neither strictly increases, or any other family's negative flips exceed its regression budget, then the Learning Harness shall reject the round, revert the change and record the reason. -6. When a round is accepted or rejected, the Learning Harness shall record attribution: which pairs flipped in the predicted family, which flipped in unpredicted families, and the resulting precision of the prediction. -7. The Learning Harness shall keep, per family, an archive of which configuration wins which pair, rather than one champion configuration. -8. If a round exceeds its cost cap, then the Learning Harness shall stop it, revert it and record the overrun. -9. The Learning Harness shall retain a model-authored skill, tool or prompt edit only after it passed the acceptance rule and was journaled with its attribution. - -### Requirement 10: Corpus repairs the loop depends on - -**Objective:** As a maintainer, I want the corpus defects that cap every number fixed before detectors are tuned against it, so that a detector change and a corpus artifact are distinguishable. - -#### Acceptance Criteria - -1. The Learning Harness shall parse TypeScript rows so that they are scorable rather than counted as unsupported. -2. When absence-labeled rows are re-harvested, the Learning Harness shall keep each handler with its decorators and registration statements, and every re-harvested excerpt shall keep its license line and pass redaction. -3. The Learning Harness shall mark the five byte-identical vibe-py twins unscorable with the reason identical twin. -4. If a unit-test fixture is strictly easier than the corpus rows its test names (fewer binding hops to the identity source, registration present when the corpus rows lack it), then the Learning Harness shall fail that check. -5. The Learning Harness shall carry a slice of pairs dated after the detector model's training cutoff and shall report it separately. - -### Requirement 11: Publishing and operations - -**Objective:** As a maintainer, I want every number regenerated from one command with its denominator, noise floor and cost, so that the roadmap cannot drift from the measurement. - -#### Acceptance Criteria - -1. When the roadmap numbers are regenerated, the Learning Harness shall produce them from one command and shall write the command, the taxonomy version, the scorable and unscorable counts, the noise floor and the cost per point beside every number. -2. The Learning Harness shall commit the raw measurement artifacts of every published number. -3. The Learning Harness shall persist every detector trajectory and every round's journal, redacted, so that a round can be replayed. -4. If a budget cap, a failing endpoint or a missing optional extra interrupts a run, then the Learning Harness shall record the reason and complete the remaining steps that do not depend on it. -5. The Learning Harness shall leave the local detection, map and pair gate outputs byte-identical to the pre-feature commit. diff --git a/.kiro/specs/learning-harness/research.md b/.kiro/specs/learning-harness/research.md deleted file mode 100644 index a192ebc..0000000 --- a/.kiro/specs/learning-harness/research.md +++ /dev/null @@ -1,137 +0,0 @@ -# Research & Design Decisions: learning-harness - -## Summary - -- Feature: `learning-harness` -- Discovery Scope: Complex Integration (extension of pairs, tool hunter, provider, regress, improve; adoption of HarnessX meta-harness; corpus repairs) -- Key Findings: - - The tool hunter already beats the overlay 12/14 vs 5/14 on the class-aware, scorable vibe-py holdout; the published 0/17 was the text-token scorer. - - Every learning path ignores the split (`evaluate_profiles`, `evaluate_mechanism_profiles`, `propose_mechanism_edits`, `evaluate_loo`, `build_pair_signals`, `export_mechanisms`); one teacher rule fixes all of them. - - The sandbox verdict is exit-code only and snippets are compile-only templates; a canary oracle is an additive callback that respects the existing safety bans. - - The chat and embedding clients share one base-URL variable; DeepSeek needs thinking disabled, `reasoning_content` replay, no `/v1`, `json_object` only, and offers no seed and no embeddings. - - HarnessX 0.1.0 already provides the round primitive (`MetaAgent.evolve`, journal with `predicted_affected` and precision discounted by unpredicted regressions); the tool supplies tasks, trajectories, the evaluator and routing. - - The field routes on coarse verifier-aligned families, gates reports on proof, and reports regression blindness in meta-agents; per-instance archives and no-regression acceptance rules are the published remedies. - -## Research Log - -### Codebase seams (2026-09-06, Explore subagent over `src/openultrasast`) -- Context: design an extension without re-reading every module. -- Findings: `run_tool_hunter(root, hotspots, *, client, model, max_steps)` with three curated tools and `suspicion` findings tagged `tool-hunter`; `ChatClient.complete(*, model, messages, tools, timeout_seconds)`; `OpenRouterChatClient.complete_chat` always sends `temperature: 0` and drops `reasoning_content`; `_hunter_matches_expected` is text-token based and every fixed-side finding leaks under `fix_policy = silent`; `run_regression(..., sandbox, sandbox_limits, images)` with `verdict_from_result` on exit code; `check_snippet_safety` bans sockets, curl and workspace writes; `MechanismStore` append-only JSONL with tombstones; `improve.run_round` journal has no hypothesis, attribution or cost fields and its regression check is per provenance profile; `load_dotenv` no-ops under pytest; `pyproject` semantic extra lacks TypeScript; `pair_gate` scores only vendored `local` pairs. -- Implications: the design adds parameters and additive fields instead of replacing modules; the family scorer is injected into the hunter path so `pairs` never imports `learning`; the oracle is an optional callback on `run_regression`; endpoint resolution takes an explicit override for tests. - -### DeepSeek platform API (2026-09-06, api-docs.deepseek.com) -- Findings: base `https://api.deepseek.com` (no `/v1` in current docs), Anthropic-shape endpoint at `/anthropic`; models `deepseek-v4-flash`, `deepseek-v4-pro`, `deepseek-v4-flash-vision-exp`; 1M context, 384K output; thinking on by default (`thinking: {"type": "disabled"}` to turn off; `temperature` ignored while thinking); tools with `tool_choice`, `reasoning_content` must be replayed on tool turns; `response_format json_object` only, occasional empty content; `logprobs`/`top_logprobs` documented, unverified in thinking mode; no `seed`, `n`, `logit_bias`; flash 0.44/1.32 USD per M in/out peak, cache hit 0.014, pro 1.32/3.96, off-peak half; concurrency 2500 flash / 500 pro; no embeddings endpoint. -- Implications: detectors and classifier run with thinking disabled and temperature 0; determinism is best-effort and the K-run noise floor absorbs it; the classifier's model tier uses `logprobs` averaged and falls to `unknown`; embeddings stay on OpenRouter; cost accounting reads DeepSeek's cache-hit usage fields. - -### Measurement that triggered the spec (2026-09-06) -- See `brief.md` Problem section and `benchmarks/measurements/2026-09-06-*`. - -## Architecture Pattern Evaluation - -| Option | Description | Strengths | Risks / Limitations | Notes | -|--------|-------------|-----------|---------------------|-------| -| Keep static machinery as the detector, add a class field | Minimal change | Cheap | Ceiling by construction; measured 5/14 vs 12/14 | Rejected | -| One generalist hunter, evolved as a whole | Single config | Simple loop | Aggregate fitness hides per-class regressions; the maintainer's oscillation observation; regression blindness in the literature | Rejected | -| Per-family detector set with auto-classifier, per-family attribution, verifier-gated reporting | This design | Matches AIxCC routing practice, GEPA per-instance archives, slice-based gating | Router error (63.8% in MoEVD); mitigated by multi-label routing plus a generalist and by measuring the router separately | **Selected** | -| Fine-tune a model per class | Weights | Potentially strong | Out of scope; label noise 25–60% in public sets; not reproducible offline | Rejected | - -## Design Decisions - -### Decision: families are the only routing and scoring key -- Context: 1.4, 3.6; text tokens and CWE ids were the proxy. -- Alternatives: fine CWE routing; mechanism ids as key. -- Selected: ten coarse families with CWE and mechanism as attributes; hierarchical relation for partial credit. -- Rationale: fine CWE from code is unsolved (≤14.7% top-1) and coarse families are what worked in AIxCC and MoEVD. -- Trade-offs: less granular reporting; compensated by attributes on findings. - -### Decision: the scorer is injected into the hunter path, not imported by `pairs` -- Context: dependency direction; `pairs` is owned by `pair-corpus-honesty`. -- Selected: `_evaluate_hunter_pair` accepts a matcher/leak callable; the CLI wires `learning.scoring`. -- Trade-offs: one more parameter; keeps `pairs` free of `learning` and the overlay path unchanged. - -### Decision: canary oracle as an optional callback on `run_regression` -- Context: 7.2; exit code cannot express "the vulnerability fired". -- Alternatives: a parallel verifier runner; parsing stderr. -- Selected: `oracle: Callable[[SandboxResult], bool] | None`; token in stdout. -- Trade-offs: minimal change to regress; templates per family live in `learning/verifiers/`. - -### Decision: HarnessX proposer behind the seam, scripted proposer first -- Context: 9.x with core `dependencies = []`. -- Selected: `Proposer` protocol; `HarnessXProposer` composes `MetaAgent.evolve` with `allowed_write_roots` = the family directory; `ScriptedProposer` for tests and phase 4 start. -- Trade-offs: two proposers; the loop is testable offline and the extra stays optional. - -### Decision: noise floor from K = 5 baseline runs is the regression budget -- Context: 8.2, 9.5; no public numeric negative-flip budget exists. -- Selected: per-family negative-flip rate between seed runs; `budget_flips = ceil(rate × scorable)`. -- Trade-offs: a noisy family gets a loose budget; reported beside the number so it is visible. - -### Decision: DeepSeek adapter wraps the existing OpenRouter client -- Context: 6.6; both clients share one base URL today. -- Selected: `ChatEndpoint` resolution order DeepSeek → OpenRouter → scripted; adapter adds `extra_body` and `reasoning_content` replay; embeddings untouched. -- Trade-offs: one adapter class; no new dependency. - -## Risks & Mitigations -- Router error routes a region to the wrong family → multi-label routing plus the generalist; router measured separately against random and oracle. -- No seed on DeepSeek → K runs and the reliable-change test; determinism recorded as best-effort. -- Canary templates cannot express some families (XSS, SSRF) → those report at `suspicion` only, stated in the taxonomy. -- Proposer confabulation → structured failure facts only; rejected buffer; length cap; structural levers preferred. -- Cost drift → per-stage metering, per-round cap with revert, cost per point published. -- Re-harvest needs network and licenses → maintainer step; excerpts re-redacted; failures leave the old excerpt and a recorded reason. - -## References -- Codebase seams and DeepSeek facts: this file, sections above. -- External literature: the four threads below. - ---- - -## External research (2026-09-06) - -Web research conducted 2026-09-06 in four threads by research subagents; summaries are theirs, condensed, with primary sources. Speculation is marked. This section backs the "What others learned" section of `brief.md`. - -## 1. Cyber reasoning systems (AIxCC finals 2025, Big Sleep, XBOW, Glasswing) - -Primary cross-team source: USENIX Security '26 SoK on AIxCC, arXiv 2602.07666. 42-b3yond-6ug and Lacrosse have no first-party write-up. - -| System | Decomposition / routing key | LLM vs tools | -|---|---|---| -| Atlantis (1st), arXiv 2509.14589, team-atlanta.github.io/blog/post-afc | N-version sub-CRSs by language (C, Java, Multilang) with orthogonal designs; six patch agents in parallel, no routing ("no single agent consistently outperformed"); rate limit per fuzzer×sanitizer; Java sinks keyed by Jazzer sanitizer class | Three tiers: LLM-augmented (seeds), LLM-opinionated (hints as optimizations, never correctness), LLM-driven (multi-agent call-graph, bug candidates, blobs). Fuzzers/concolic are ground truth | -| Buttercup (2nd), blog.trailofbits.com 2025-08-08/09 | Seven patch agents by role (RCA, retrieval, writer, reflection); language tooling and CWE guidance in prompts | Fuzzers discover; tree-sitter/CodeQuery program model; cheap non-reasoning LLMs only ($21k for 100k calls); >90% accuracy | -| RoboDuck (3rd), theori.io/blog/aixcc-and-roboduck-63447, /building-effective-llm-agents-63446 | Bug candidate is the anchor: Infer + LLM static analysis + crashes → logprob-scored binary classifier (~$0.001/report) → top 20% to a source-browsing agent (~$0.50) | LLM-first PoVs without fuzzing; three models race, first verified wins; Infer ~99.9% FP | -| FuzzingBrain (4th), arXiv 2509.07225 | 23 strategies keyed on mode × language × stage; per-language CWE lists aligned to sanitizers | LLM-dominant; libFuzzer only; nothing reported unless a sanitizer fires on a real build | -| ARTIPHISHELL (5th), support.shellphish.net 2025-08-22 postmortem | 53 components; multi-source voting; dual patchers | PoV submitted only after five consecutive local reproductions | -| Big Sleep / Naptime, projectzero.google 2024/06 and 2024/10 | No classifier; variant analysis seeded from recent commits | Single agent with code browser, Python, debugger, reporter; must reproduce a crash; parallel independent trajectories | -| XBOW, xbow.com/blog/top-1-how-xbow-did-it | Coordinator spawns per-vulnerability-class solvers; validators keyed by class (canary, headless browser for XSS, timing for blind SQLi) | The non-AI validator "is the only one who can decide if the agent succeeded" | -| Anthropic Glasswing, anthropic.com/research/glasswing-initial-update | Threat-model builder prioritizes targets; scanning subagents; reusable skills | 90.6% of 1,752 findings valid after human reproduction; internals unpublished | - -Lessons: (1) route on mode × language × sanitizer/CWE family, never fine CWE; route to what the verifier can check. (2) Only proof survives; the accuracy multiplier (Atlantis 0.9999 vs 0.9044) decided the ranking; RoboDuck lost 16.3 points to FPs. (3) Cheap classifier → expensive agent → verifier (RoboDuck 500× cost tiering; logprob averaging for stable scores). (4) Ensembles beat selection when the winner is unpredictable; attribution per agent done by hand post hoc. (5) Dedup granularity is a scoring bug (Shadowsocks: five same-pattern bugs differing by line). (6) Validated ≠ correct: 38–46% of automatically accepted patches were semantically wrong (SoK KF4). (7) Hallucination controls are structural: forced tool calls, curated tools instead of bash, symbol resolution via Joern/LSP. (8) Stability beat cleverness: budget caps, cached container state, exit-code-0 logging, lost logs sank teams. (9) Cross-run learning nearly absent; Atlantis's only trained component (GRPO retrieval, C only) showed catastrophic forgetting across projects; daily leakage-resistant CI benchmarks instead. (10) Cost is a control: >$1,000/hour unbounded (arXiv 2603.08566); Buttercup $181/point. - -## 2. Self-improving harness methods - -| Method | Evolved unit | Search operator | Validation | -|---|---|---|---| -| DSPy MIPROv2 (dspy.ai) | Few-shot demos + instructions per module | Bootstrapped traces; LLM proposals; Bayesian optimization | Minibatch trials, full validation every 5 steps; ≥200 examples advised | -| GEPA, arXiv 2507.19457 | One module's prompt per iteration | Natural-language reflection on trajectories; system-aware merge | Minibatch gate → full per-instance Pareto front; candidates sampled ∝ instances led | -| TextGrad; PrefPO critique arXiv 2603.19311 | Prompt text | Textual gradients | Validation revert; 86% of TextGrad prompts hacked/brittle, 14.7× length inflation | -| OPRO 2309.03409 / EvoPrompt 2309.08532 / PromptBreeder 2309.16797 | Task prompt | Proposal / GA / self-referential mutation | Train-score selection; EvoPrompt degrades to generic prose on structured tasks | -| AlphaEvolve (DeepMind blog 2025) | Code diffs | LLM mutation over MAP-Elites program DB | Evaluator cascade; needs automatically verifiable problems | -| Darwin Gödel Machine, arXiv 2505.22954, sakana.ai/dgm | Agent code/tools | Archive sampling ∝ score, ∝ 1/(1+children) | Staged 10→60→200 tasks; held-out models/languages; faked test logs when checker visible | -| ADAS, arXiv 2408.08435 | Agent code | Meta-agent over archive | Held-out validation and domains | -| Reflexion / ExpeL 2308.10144 / AWM 2409.07429 / Voyager 2305.16291 | Reflections / insights / workflows / skills | Self-critique; contrast; induction | Mostly none | -| Self-Harness, arXiv 2606.09498 | Harness components | Weakness mining → K minimal proposals | Accept iff Δ_in ≥ 0 ∧ Δ_ho ≥ 0 ∧ one > 0 | -| SkillOpt, arXiv 2605.23904 | SKILL.md | Reflection with rejected-edit buffer | Strict improvement on held-out; ties rejected | -| AHE, arXiv 2604.25850 | Tools, middleware, memory, prompts | Evidence-driven edits with per-edit manifest (predicted fixes and at-risk regressions) | Prediction vs observed deltas; rollback | -| Meta-Harness, arXiv 2603.28052 | Harness code | Agentic proposer over prior candidates and traces | Held-out models and tasks | - -Lessons: (1) per-instance/per-class tracking doubled gains vs greedy best-overall (GEPA +12.44% vs +6.05%). (2) No-regression acceptance rule; reject ties (Self-Harness, SkillOpt). (3) Regression blindness: AHE fix-prediction precision 33.7% but regression prediction 11.8%; the cross-class sweep is mandatory. (4) Two-stage evaluation is where sample efficiency comes from; most budget is validation. (5) Prompt bloat is the overfit signature (Decagon: +75% length, −2% accuracy; cap ~1,500 chars gave 4× compression at −0.8%). (6) Keep the reflector blind to validation; keep a rejected-edit buffer (arXiv 2602.22483; SkillOpt). (7) Structural edits transfer (+5.1–10.1 points across model families), system-prompt-only edits regressed −2.3 (AHE ablation). (8) Self-diagnosis confabulates: 0/121 Reflexion reflections correct; structured trajectory facts → 86% (arXiv 2605.29463). (9) Hide the checker (DGM). (10) One objective per round (arXiv 2605.26046: joint criteria −59% task focus). (11) Audit that a skill beats re-showing the failure (ContinualSkillBench arXiv 2608.03874). (12) Harness-edit quality flat across model tiers; spend on the detector (arXiv 2605.30621). Speculation: no source measures class-to-class oscillation; per-class Pareto retention plus the acceptance rule should bound it. Anthropic's Claude Code postmortem (InfoQ 2026-05: three unattributed harness changes → six-week degradation) is the cautionary tale. - -## 3. Dataset pitfalls and class taxonomies - -Scoring: (1) dedup by normalized body hash before splitting; split by commit, chronologically (PrimeVul arXiv 2403.18624; test-set copies 12.7–18.9% in older sets). (2) Automatic labels 25–60% correct (BigVul 25%, CVEfixes 51.7%, DiverseVul 60%; Croft et al. ICSE'23 20–71% wrong, 17–99% duplicates; CleanVul arXiv 2411.17274). (3) Pair-wise outcomes P-C/P-V/P-B/P-R, not F1 (GPT-4 CoT P-C 12.94% vs P-V 54.26%; JitVul ACL 2025: agents lose 8–9 points of pair accuracy while "winning" F1; Vul-RAG pairwise 0.21 vs plain 0.61). (4) Directional-bias index (CWE-Trace arXiv 2606.20502: −85.5 to +94.8 points at ~50% accuracy). (5) Score at fixed FPR (PrimeVul VD-S; VFC benchmark arXiv 2605.13138). (6) Match on location and class with hierarchical credit and hallucination penalty (CASTLE arXiv 2503.09433; ALPHA arXiv 2601.01320). (7) Robustness to renames/whitespace/dead code (SecLLMHolmes arXiv 2312.12575; patched-code FPs dominate). (8) CVE-date contamination checks are weak evidence, keep a post-cutoff slice anyway (thin: one paper). (9) Function-level pairs lose context: up to 54% of patched-code misclassification (arXiv 2504.13474). (10) Run-to-run consistency 8.3–81.9% by model (ALPHA). - -Taxonomy: (1) fine CWE from code unsolved (top-1 ≤ 14.7% over 74 CWEs; 86.5% lateral errors; VulDetectBench arXiv 2406.07595 >80% coarse, <30% detailed). (2) From CVE text ~85–90% ceiling (arXiv 2603.14911; CweAgent arXiv 2608.21977). (3) Human/NVD disagreement 15–50%; CWE-699 families as "defensible" (Seclometry; Sonnet vs NVD 73.1% exact, 84.5% with parent–child, LLM right in 72/100 disagreements). (4) Score hierarchically. (5) Few aggregated classes: MoEVD arXiv 2501.16454 collapsed 88 → 12 families, per-class experts +12.8% F1, router 63.8% right, F1 0.18 when wrong; three-class OOB/UAF/other 87.4% (arXiv 2509.22796). (6) Specialists need hard negatives (arXiv 2408.02329: F1 0.89 in class → 0.12–0.19 on full corpus). (7) Per-class detectors work when the class fixes sources/sinks (IRIS arXiv 2405.17238: 55 vs 27 CodeQL detections, FDR 46–85%). (8) VFC typing immature (F1 0.53–0.70). Speculation: 8–12 hierarchy-derived families with partial credit beats fine CWEs for this corpus. - -## 4. Routing and regression control - -Router: (1) learned routers plateau 10–20 points below oracle, sometimes lose to random (RouteLLM arXiv 2406.18665; RouterBench arXiv 2403.12031; arXiv 2606.07587). (2) Evaluate router AUROC separately from expert strength (arXiv 2602.11877). (3) Small embedding classifier fine-tuned on own labels: 80.4% → 98.5% (Red Hat vLLM Semantic Router, 2026-06). (4) Cascades tolerate router error until judge error > 0.2 (FrugalGPT arXiv 2305.05176; RouterBench). (5) Multi-expert routing with balance (Expert-Choice, Google; AgentRouter arXiv 2510.05445); speculation: multi-label with `unknown` is safer than argmax for SAST. - -Attribution: (6) slice everything (Oakden-Rayner arXiv 1909.12475; Slice-based learning 1909.06349; Overton 1909.05372; Model Cards 1810.03993). (7) Negative flips per item (PCT arXiv 2011.09161: 5.2% NFR at equal accuracy; MUSCLE 2407.09435: 8–49% NFR across Llama versions; arXiv 2604.27405: +1.6–2.8 point gains masked 28–39% items worse, single-shot eval missed 42% of changes → K ≥ 3 samples and a reliable-change test; UpgradeBench 2608.20918: neutral retraining breaks 1–5%). (8) Frozen per-class configurations (+20%, Progressive Prompts arXiv 2301.12314); self-feedback-only skill loops drift, external feedback does not (SkillLearnBench 2604.20087); self-revision flips correct → incorrect more often than the reverse (arXiv 2310.01798). (9) Per-instance archives over one champion (GEPA; DGM lineage logs caught faked tests). (10) Conjunctive per-slice gates with relative thresholds vs the last blessed baseline (TFMA model validations; ease.ml/ci arXiv 1903.00278; Amazon Alexa regression-free updates). No public numeric NFR budget; speculation: the measured seed-noise floor per class is the defensible budget. diff --git a/.kiro/specs/learning-harness/spec.json b/.kiro/specs/learning-harness/spec.json deleted file mode 100644 index 2c8cc8f..0000000 --- a/.kiro/specs/learning-harness/spec.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "feature_name": "learning-harness", - "created_at": "2026-09-06T06:32:41Z", - "updated_at": "2026-09-08T00:00:00Z", - "language": "en", - "phase": "implementation", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true, - "depends_on": [ - "pair-corpus-honesty", - "corpus-seeded-mechanisms", - "authorization-obligations", - "tree-sitter-overlay-extra", - "harnessx-self-improving-rulesets" - ], - "approval_note": "2026-09-06 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); 2026-09-06 design approved by the maintainer in session ('approve the design and generate the tasks'); 2026-09-06 tasks approved by the maintainer in session ('approve the tasks and go on with the implementation'); 2026-09-06 requirements amended by the maintainer in session ('amend the requirements to untrusted_destination and note the flat taxonomy'): Req 1.1 names untrusted destination instead of server-side request forgery, and Req 2.7 and 3.7 require every agreement and family figure to state that hierarchical partial credit could not apply while the taxonomy is flat; 2026-09-06 Req 6.1 amended by the maintainer in session ('drop the skills set from the requirement'): a detector configuration has no skills set, because the skill router belongs to the tool's earlier shape and a family's prompt, checklist, tools and counterexamples already carry what a round evolves; 2026-09-06 design amended during task 4.1 (recorded in design.md): a recipe may name cross-file registration documents, because connexion registers by operationId in an OpenAPI file and express in a router module, so Req 10.2 is unreachable from a single-file excerpt for those rows; 2026-09-06 task 5.3 amended by the maintainer in session (asked and answered): the first evolve rounds target `injection` rather than `access_control`, because under the split and review-tier rules access_control has one train teacher and four holdout pairs and teaches zero obligation shapes, while injection has twelve teachers and ten holdout pairs; 2026-09-08 SUPERSEDED by execution-grounded-detection: this session's measurements showed the noise architecture (K-runs, per-family floor, regression budget, sign test, evolve loop) is a statistical treatment for a structural problem, and that the field (Big Sleep, AIxCC finals, Clearwing) grounds detection in execution rather than in scoring against a labeled corpus. The taxonomy, the IR candidate enumerator (96.6% ceiling) and the obligation checker survive as the web/logic detection path to contribute to a Clearwing-based substrate.", - "superseded_by": "model-grounded-detection" -} diff --git a/.kiro/specs/learning-harness/tasks.md b/.kiro/specs/learning-harness/tasks.md deleted file mode 100644 index 6dc3a37..0000000 --- a/.kiro/specs/learning-harness/tasks.md +++ /dev/null @@ -1,269 +0,0 @@ -# Implementation Plan - -- [x] 1. Foundation: taxonomy, endpoint, corpus fields, grammar -- [x] 1.1 Closed family taxonomy with a verifier per family - - Ten families exactly as decided, each with its CWE set, mechanism set, verifier kind (canary, static, none) and description; `unknown` always present with no verifier; a version string. - - Loader rejects an unknown field, an eleventh family, a missing verifier kind or a CWE claimed by two families, naming the offender; relation lookup answers same, parent–child, lateral or fabricated for any two ids. - - Observable: the taxonomy loads from the ruleset directory, a fixture with a fabricated family fails by name, and every mechanism id in the mechanism vocabulary resolves to exactly one family. - - _Requirements: 1.1, 1.2, 1.3, 1.4_ - -- [x] 1.2 Independent chat endpoint with the DeepSeek adapter - - Endpoint resolution order: explicit override, scripted client modes, DeepSeek key and base URL, then OpenRouter; the embedding client keeps reading only its own variables; `[models]` gains the hunter and judge defaults and the endpoint keys, `[learning]` gains its section. - - Adapter disables thinking and sends temperature zero for detector and classifier calls, keeps the provider's reasoning field on assistant messages so a caller can replay it, supports JSON-object mode with one retry on empty content and optional log-probabilities, and reports normalized usage and cost from the provider's cache-hit fields. - - Observable: with both provider keys set the hunter client targets the DeepSeek base URL without a version suffix while the embedding client still targets OpenRouter; a scripted client is chosen under the existing environment flag; a recorded request shows thinking disabled and the reasoning field retained on the reply. - - _Requirements: 6.6, 11.4_ - -- [x] 1.3 Additive corpus fields and hygiene at load - - Expected rows accept an opaque family id (validated later by the classifier task, so the corpus module never imports the learning package); a pair carries a computed unscorable reason (identical twin by normalized body comparison with headers stripped, computed only when the catalog does not already declare a known limit) and is never dropped; duplicates across pairs are reported before any split; a repository-and-date split helper assigns train or holdout to rows that lack a split, keeps declared splits, and reports rows from one repository that straddle the split; pair signals take a split filter. - - Observable: the five identical vibe-py twins load with the identical-twin reason and the existing catalogs load with unchanged row counts; the straddle report on the current catalogs is produced and committed. - - _Requirements: 4.1, 4.2, 4.3, 4.4_ - -- [x] 1.4 TypeScript rows parse under the extra - - The semantic extra gains the TypeScript grammar and the parser maps TypeScript and TSX to it; the extra-free suite still skips them with the existing marker. - - Observable: every vendored TypeScript pair parses on both sides under the extra, and the parser reports no unsupported-language result for TypeScript or TSX. - - _Requirements: 10.1_ - -- [x] 1.5 Curated hunter tools and hunter-loop parameters - - Four new repository tools beside the existing three: resolve a definition by symbol, list entry points, list flows for a function, list obligations for a function, each clamped to the scanned repository and returning identifiers and line spans only; the hunter loop gains prompt, context files, tool set, character limit and tag parameters with today's behavior as defaults and replays the provider's reasoning field across tool turns. - - Observable: the existing hunter tests pass unchanged with default parameters; each new tool answers on the shared handler fixture and refuses a path outside the repository; a recorded tool-turn conversation shows the reasoning field replayed. - - _Requirements: 6.4, 6.5_ - -- [x] 2. Core: classifier, scorer, detectors, verifiers, journal -- [x] 2.1 (P) Classifier tiers - - Tier one decides from rule id, sink family, obligation kind, mechanism id or labeled CWE without a model; tier two asks the chat endpoint in JSON-object mode with thinking disabled and averages log-probabilities, falling to unknown under a threshold; tier three is unknown; every answer records its tier; results are multi-label; loaded catalog family labels are validated against the taxonomy here and a fabricated family is rejected by name. - - Observable: on the shared handler fixture tier one classifies the leaky handler as access control without any client; with a scripted client an off-vocabulary answer becomes unknown; a catalog with a fabricated family is rejected by name. - - _Requirements: 2.1, 2.2_ - - _Boundary: Classifier_ - -- [x] 2.2 Classifier measurement and review queue - - Report: agreement with maintainer labels on reviewed pairs with parent–child partial credit and no credit for lateral or fabricated ids, confusion matrix per family, accuracy against a random router and an oracle router, counts per tier and unknown; disagreements append to a review queue and never change a label; the whole corpus, including free-text and `other` rows, is classified with per-family counts. - - Observable: the report on the vendored corpus lists the vfc `other` rows by resulting family and the queue holds every disagreement once with both answers and the tier. - - Observable: the report also states whether hierarchical partial credit could apply. - - _Requirements: 2.3, 2.4, 2.5, 2.6_ - -- [x] 2.3 (P) Per-pair family scoring - - Detection only for a finding whose family tag relates same or parent–child to the labeled family and whose line lies in the labeled function, with the maximum penalty for a fabricated family; silence only when no such finding lies in the labeled function on the fixed side; other findings counted separately, never as leaks; text is never inspected; outcomes pair-correct, both-flagged, both-silent, reversed and unscorable; the unscorable reason also covers unsupported language, unresolved label and missing context. - - Observable: a synthetic run set where every fixed side carries an other-family finding scores full silence; a finding with a fabricated family never counts; a pair in an unsupported language scores unscorable with that reason. - - _Requirements: 3.1, 3.2, 3.6, 4.1_ - - _Boundary: Scorer_ - -- [x] 2.4 Aggregation, noise handling and denominators - - Per family and slice: outcome counts, recall, silence, Youden, directional-bias index, a fixed-false-positive-rate view; K runs per pair with the majority outcome, negative flips per family against a baseline set, and a paired reliable-change test; unscorable rows listed by reason and excluded from every denominator; every metrics block carries the taxonomy version and the scorable and unscorable counts. - - Observable: a single-run difference below the reliable-change threshold reports no change while a consistent flip across runs does; the unscorable count and taxonomy version appear in every metrics block. - - Observable: every metrics block states whether hierarchical partial credit could apply. - - _Requirements: 3.3, 3.4, 3.5, 3.7, 4.5_ - -- [x] 2.5 (P) Per-family detector configurations and the family detector runner - - A configuration directory per family: prompt under a stated length cap, checklist, tool list from the curated set only, step and cost budget, maximum characters, hard negatives and counterexamples, version; the loader rejects an over-cap prompt, an unknown tool or a missing family; this task ships the loader and a test fixture configuration, the shipped per-family directories are written by round zero. - - The runner takes the families admitted by the classifier as input, builds the detector's user message from the whole file plus definitions its tools resolve, runs the hunter loop with the family's prompt, tools and budgets, and tags each finding with exactly one family tag and one detector-version tag; the generalist is the unknown family's configuration. - - Observable: a family run on the shared fixture with a scripted client yields findings tagged with that family and version and uses only the family's tool list; a configuration whose prompt exceeds the cap fails to load by name. - - _Depends: 2.1_ - - _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5_ - - _Boundary: Detectors_ - -- [x] 2.6 (P) Verifier registry and the oracle callback - - The regress verdict accepts an optional oracle callback and keeps exit-code behavior otherwise; a registry keyed by the taxonomy's verifier kind returns a canary, static or no-op verifier; the static verifier raises access-control claims to static corroboration when the obligations checker reports the same function; the no-op verifier leaves suspicion. - - Observable: with the fake sandbox runner a result whose output carries the token yields triggerable while a nonzero exit without the token does not; a family with no verifier keeps suspicion on the shared fixture; the access-control claim on the leaky handler reaches static corroboration. - - _Requirements: 7.1, 7.3, 7.4_ - - _Boundary: Verifiers, Regress_ - -- [x] 2.7 Canary templates and the second judge - - Family snippet templates for injection, file path and deserialization plant a canary (row in an in-memory database, file under the scratch mount, marked object) and print the token only when the vulnerability fires; the second judge asks the judge model with the snippet, oracle output and claim and answers confirm or not in JSON-object mode; a claim is published as proven only when it confirms; templates, oracle code and gold labels live outside every proposer write root. - - Observable: every canary template passes the existing snippet safety check; a scripted judge that declines keeps the claim below proven; templates and oracle code live under the verifier directory and gold labels under the corpus directory. - - _Requirements: 7.2, 7.5_ - -- [x] 2.8 (P) Round journal, archive and rejected buffer - - Append-only round records with hypothesis, levers, predicted affected and at-risk families, outcome, reason, target deltas, per-family sweep flips, attribution (flipped predicted, flipped unpredicted, precision), cost, taxonomy and configuration versions; a per-family archive of which configuration version wins which pair; a rejected-hypothesis buffer per family; every persisted text passes redaction. - - Observable: a record round-trips through the file; the buffer returns only the target family's rejected hypotheses; the archive names one winner per pair per family; a secret literal written into a record is stored redacted. - - _Requirements: 9.3, 9.6, 9.7, 11.3_ - - _Boundary: Journal_ - -- [x] 3. Integration: split, scoring paths, rounds, proposer, publish, CLI -- [x] 3.1 One teacher rule across every learning path - - Teachers are vendored, gating-tier, train-split, scorable pairs; a refusal names any candidate taught by or touching a holdout pair; the exporter, leave-one-out, per-profile evaluation, mechanism-profile evaluation, mechanism admission and pair signals all obtain teachers through the rule and record refusals as degradations; the mechanism lever's holdout number is re-measured under the rule and its raw artifact committed for the publish task. - - Observable: leave-one-out on the toy slice seeds from train pairs only; a candidate shape taught by a holdout pair is refused by name; the re-measured artifact is in the measurements directory. - - _Depends: 1.3_ - - _Requirements: 5.1, 5.2, 5.3_ - -- [x] 3.2 Family scoring on the pairs hunter path - - The pair scorer's hunter path takes an injected family matcher and leak rule from the learning package, classifies each pair once, runs the admitted family detectors plus the generalist K times per side, and emits family outcomes and metrics; the overlay and inventory paths stay untouched. - - Observable: the pairs command with a scripted client on the shared fixture reports a family outcome per pair; no text-token matching remains on the hunter path; the pair gate output is byte-identical to before. - - _Depends: 2.1, 2.4, 2.5, 2.6_ - - _Requirements: 3.1, 3.2, 3.3, 3.5, 11.5_ - -- [x] 3.3 Round zero and the noise floor - - Clone the current hunter prompt into every family configuration; run each family K = 5 times over the scorable vendored pairs of the selected slices at deterministic settings; record the per-family negative-flip rate between runs as that family's regression budget; persist noise floors and round-zero artifacts keyed by detector model so another model can be baselined and compared without touching configurations; the web families over vibe-py and agent-vfc are the default selection and the memory-safety family over vfc is selectable but never gates. - - Observable: with a scripted client round zero writes a noise-floor file with one budget per family and a round-zero report per slice; a second baseline under another model name writes a separate artifact and the comparison table lists both. - - _Depends: 3.2_ - - _Requirements: 8.1, 8.2, 8.3, 8.4_ - -- [x] 3.4 Family detectors, verifiers and tags in scans - - In a standard scan each classified region runs its admitted family detectors plus the generalist from the directories round zero wrote (a missing family directory is a recorded degradation and that family's detector is skipped), claims pass through the family's verifier, findings carry family, detector and verifier tags in reports and SARIF, and the manifest gains a learning block with taxonomy version, per-family counts, the classifier report and the current round; the detection and map gates stay byte-identical. - - Observable: a standard scan of the shared fixture with a scripted client writes findings tagged with a family, a detector version and a verifier rung, and the manifest block; the detection and map gate outputs diff empty. - - _Depends: 3.3, 2.6_ - - _Requirements: 6.3, 7.1, 11.5_ - -- [x] 3.5 Proposer protocol, scripted proposer and the HarnessX proposer - - Failure facts for one family: misses and leaks with pair, function, sink or check, line, verifier outcome and run outcomes, plus the rejected buffer and the current configuration; a proposal is one hypothesis, one lever (tool, checklist, counterexample, memory, prompt), one file-level change inside the family directory, and predicted affected and at-risk families; the proposer never receives holdout inputs, outputs or scores. - - A scripted proposer for tests and offline rounds; a HarnessX proposer composing the meta-agent with write roots limited to the family directory, degrading to a recorded reason without the extra. - - Observable: the scripted proposer returns a proposal from a facts fixture; the HarnessX proposer is skipped with the reason recorded when the extra is absent and, when present, writes only inside the family directory; the write-root list excludes the verifier and corpus directories. - - _Depends: 2.5, 2.8_ - - _Requirements: 9.1, 9.2, 5.4, 7.6_ - -- [x] 3.6 Acceptance rule and directory revert - - Acceptance requires train and holdout non-negative with one strictly positive and every other family within its budget; ties reject; a family directory snapshot restores byte for byte on rejection or overrun; refusals for holdout contact, length cap and lever set are outcomes with reasons. - - Observable: a helpful change is accepted, a tie is rejected, a change that breaches another family's budget is rejected with the family named, and a revert restores the directory byte for byte. - - Observable: every family other than the target has a byte-identical directory before and after the round, whatever the outcome. - - _Depends: 3.3_ - - _Requirements: 6.2, 9.5_ - -- [x] 3.7 Evolve round orchestration, cost and journal - - One round: facts, proposal, refusals, train minibatch, target-family holdout, sweep over every other family's holdout, K-run scoring at every stage as the scorer requires, accept (freeze, bump version, archive) or revert; cost metered per stage with revert and a recorded reason on overrun; a journal record and the rejected buffer written for every outcome; a round directory holds the snapshot, proposal, scores and trajectories and a repeated round number is refused. - - Observable: with the scripted proposer each of accepted, rejected, budget-breach and cost-overrun leaves one journal record with attribution and a round directory; a repeated round number is refused. - - The round directory holds the pair identities behind the attribution counts and every detector trajectory, redacted, so a round can be replayed and its attribution audited; every stage scores at least three runs per pair. - - _Depends: 3.5, 3.6_ - - _Requirements: 3.5, 9.4, 9.6, 9.7, 9.8, 9.9, 11.3_ - -- [x] 3.8 Publish from artifacts - - One entry point regenerates every family and slice number from committed artifacts, writes a markdown table with scorable and unscorable counts, taxonomy version, noise floor and cost per point beside every number, replaces the roadmap's measured section between markers, and writes the raw artifacts under the measurements directory. - - Observable: running the publish entry point twice without new artifacts produces byte-identical output; the corrected mechanism-lever number from the re-measurement artifact replaces the earlier roadmap value. - - _Depends: 3.1, 3.3_ - - _Requirements: 11.1, 11.2, 4.5, 5.3_ - -- [x] 3.9 Learning subcommands - - Subcommands for classify, score, baseline, round and publish with the flags the design names; the pairs hunter path gains the K-runs flag; every step runs under the harness runtime with degradations for missing extras, endpoints or sandbox. - - Observable: each subcommand exits zero on the shared fixture with a scripted client and writes its artifact; the roadmap section written by the publish subcommand carries the command line that produced it; a missing endpoint records the reason and the remaining steps complete. - - _Depends: 3.7, 3.8_ - - _Requirements: 11.1, 11.4_ - -- [x] 4. Corpus repairs -- [x] 4.1 Re-harvest absence rows with registration context - - Absence rows in vibe-py and agent-vfc gain their family and the handler-context mode in the recipes; the maintainer re-harvests them over the network; every re-harvested excerpt keeps its license line, passes redaction and keeps the decorators and registration statements; failed fetches keep the old excerpt with a recorded reason; catalogs regenerate idempotently. - - Observable: the vampi and threatbyte excerpts contain their registration lines, the entry-point mapper names their handlers, and the catalogs load with unchanged vendored counts. - - _Depends: 1.3_ - - _Requirements: 10.2_ - -- [x] 4.2 Identical twins and the fixture-difficulty check - - The five identical vibe-py twins carry the identical-twin known-limit reason in the catalog; a test derives a difficulty vector for a unit fixture (binding hops to the identity source, registration present, cross-file registration) and for the corpus rows its docstring names, and fails when the fixture is strictly easier. - - Observable: the twins are excluded from every denominator with their reason listed; the shared obligations fixture is compared against the re-harvested vampi rows and the check's verdict is recorded. - - _Depends: 4.1_ - - _Requirements: 10.3, 10.4_ - -- [x] 4.3 Fix dates and the post-cutoff slice - - Catalog rows gain a fix date populated by the catalog generator from recipe metadata where available; rows without a date are counted separately; rows dated after the configured detector cutoff form a separately reported slice in every scoreboard. - - Observable: the scoreboard shows the post-cutoff slice with its own counts, the cutoff date it used and the undated count. - - _Depends: 1.3, 2.4_ - - _Requirements: 10.5_ - -- [ ] 5. Validation -- [x] 5.1 Gates, matrices and degradation paths - - Detection, map and local pair gate outputs byte-identical to the pre-feature commit; the extra-free suite green with semantic and HarnessX tests skipping; each optional dependency's absence exercised once with its recorded reason. - - Observable: the three gate outputs diff empty against the recorded baseline and both test matrices pass. - - _Requirements: 11.4, 11.5_ - -- [ ] 5.2 Live round zero - - Round zero through the DeepSeek endpoint on the vibe-py and agent-vfc web families with K = 5, plus the memory-safety family over the vfc slice as a measured, non-gating run; artifacts, noise floors and cost per pair committed; the classifier report and review queue produced on the same run; a second baseline with the judge model on one slice for the model comparison table. - - Observable: the published table shows per-family outcomes, noise floors and cost for both models and for the vfc memory-safety family, and the roadmap section is regenerated from it. - - **Blocked 2026-09-06: no `DEEPSEEK_API_KEY` in `.env` or the environment.** Everything else is ready; the run is one command per baseline. The maintainer chose to add the key and have the full run go ahead (web families with `deepseek-v4-flash`, the vfc memory slice as a measured non-gating run, and a second baseline with `deepseek-v4-pro` on one slice). Scale, measured on the corpus: 58 scorable web pairs plus 176 vfc pairs, both sides, K=5 = 2,340 detector runs at up to four model calls each; the design estimated under $15 at peak flash pricing. - - Dress-rehearsed offline 2026-09-06 with the scripted client over the whole vibe-py slice at K=3: round zero produced six families with per-slice blocks, correct denominators (the twins out by reason), and a floor each; a round then read the recorded proposals, applied one, swept the other five families for negative flips (0 each), rejected on `tie`, reverted the family directory byte for byte and archived the rejected version's per-pair results. The only thing the live run adds is a detector that can actually see. - - _Depends: 3.9, 4.1_ - - _Requirements: 8.3, 8.4, 11.1, 11.2_ - -- [ ] 5.3 First evolve rounds - - **Amended 2026-09-06 (maintainer, in session): the target family is `injection`, not `access_control`.** Measured on the corpus under the split and review-tier rules: access_control has **1 train teacher and 4 holdout pairs**, six of its rows unusable because every agent-vfc row sits at tier `title`, and the vibe-py absence rows that could teach are Real-Vuln-Benchmark trap pairs whose two sides are different handlers — so the train split teaches **zero** obligation shapes and any accepted change would rest on one pair. `injection` has 12 train teachers and 10 holdout pairs, the only family in the corpus with enough train evidence to tell a real change from noise. The first memory entries are therefore the mechanism shapes the train split really teaches rather than obligation shapes, which do not exist yet. access_control follows when its corpus can teach: either a maintainer reviews agent-vfc rows up to `reviewed`, or the trap rows gain non-trap twins. - - Three rounds with the scripted proposer on the injection family using the shapes the train split teaches as the first memory entries, then one round with the HarnessX proposer; every round journaled with attribution; the roadmap regenerated. - - Observable: the journal holds four records with outcomes and attribution precision, at least one accepted change is versioned in its family directory, and no other family exceeded its budget in any accepted round. - - _Depends: 5.2_ - - _Requirements: 9.9_ - -## Task-plan review notes - -- Sanity review round 1 (2026-09-06): NEEDS_FIXES, eleven findings, all repaired (K-run policy honors Req 3.5; per-model baselines added to the design; family validation moved out of the corpus loader; reasoning replay owned by the hunter loop; Req 5.3 split into re-measure and publish; scoring path split into pairs and scan tasks; oversized tasks split; fix-date prerequisite added; fixture-difficulty check depends on the re-harvest; memory-safety family measured on vfc; detector task depends on the classifier and round zero owns the shipped directories; observables re-anchored). -- Sanity review round 2 (2026-09-06): NEEDS_FIXES with three local items, repaired without a third review under the bounded loop: task 1.5 delivers the four curated tools and the hunter-loop parameters that 2.5 relied on; the scan-path task now follows round zero and degrades when a family directory is missing; the write-root observable moved to the HarnessX proposer task with 7.6. Non-blocking notes applied: the split helper assigns as well as reports (4.4), 3.2 depends on 2.1 rather than 2.2. - -## Implementation Notes - -- Requirements approved 2026-09-06, design approved 2026-09-06, tasks approved 2026-09-06 (maintainer, in session). -- Task 1.1 (2026-09-06, RED first: 9 failing tests in `tests/test_learning_families.py`, `ModuleNotFoundError: openultrasast.learning`; then green): `learning/families.py` and `ruleset/families.toml` with the ten decided families, each carrying its CWE set, mechanism set, verifier kind and description. Decisions taken while writing the data: (a) `unknown` carries the six mechanisms that span families (`source_reaches_sink`, `container_taint`, `validation_strength`, `redos`, `cross_artifact`, `other`) and the two weaknesses we deliberately do not target (CWE-20 generic validation, CWE-835 infinite loop), so a row we considered and declined is visible instead of unresolved; an earlier draft said `unknown` had none. (b) The maintainer's "server-side request forgery" family is named `untrusted_destination` and also covers open redirect (CWE-601), because both are untrusted input choosing a request destination and the corpus carries 34 open-redirect rows that would otherwise be unclassifiable. (c) The taxonomy is flat: `parent` is an optional field no shipped family uses, exercised by a fixture, because Req 2.3 and 3.1 need a parent-child arm and inventing a hierarchy would be worse than leaving it empty. (d) Abstention earns no partial credit: `related("unknown", X)` is `lateral`. All 33 labeled corpus weaknesses resolve, none is claimed twice, and every mechanism id belongs to exactly one family. -- Task 1.2 (2026-09-06, RED first: 8 failing tests in `tests/test_learning_endpoint.py`; then green): `learning/endpoint.py` resolves override, then the scripted flag, then a configured endpoint, then DeepSeek, then OpenRouter; the embedding client is untouched. `provider/openrouter.py` gains `complete_chat_raw` (whole payload, for `usage`) and an `extra_body` passthrough; `ChatResponse` gains `reasoning` and `mean_logprob`. Design deviation: `ChatEndpoint` deliberately holds no `api_key` (the design listed one) so the record is safe to journal; the key stays on the client. `[models]` gains `judge`, `chat_base_url`, `chat_api_key_env`; `[learning]` is new. `ModelConfig.hunter` keeps its `None` default on purpose: defaulting it would silently enable LLM stages in existing scans, so the DeepSeek model defaults live in `resolve_models`. -- Task 1.3 (2026-09-06, RED first: 7 failing tests in `tests/test_pair_hygiene.py`; then green): `ExpectedFinding.family` (opaque; the classifier validates it), `PairCase.unscorable`, `PairCase.split_declared`, `PairCase.fix_date` (the generator fills it in 4.3), `duplicate_groups`, `split_by_repository` with `SplitReport`, and a split filter on `build_pair_signals`. Measured and committed as `benchmarks/measurements/2026-09-06-corpus-hygiene.json`: **19 repositories straddle the train/holdout split** (a learning path can be taught by one file of a repository and evaluated on another) and **10 groups of pairs share a vulnerable excerpt byte for byte**, mostly the same fix harvested twice at different granularity. Only three vendored rows (the `local` slice) declare no split. -- Task 1.4 (2026-09-06, RED first: 2 failing tests in `tests/test_typescript_grammar.py`; then green): the semantic extra gains `tree-sitter-typescript`; `grammar_for` maps a language to (module, attribute) because the wheel ships two grammars, and `parse_with_cst` picks the JSX-aware one for a `.tsx` path while the parsed file keeps the language it was given. Eighteen agent-vfc rows plus the TypeScript absence rows now parse on both sides. -- Task 1.5 (2026-09-06, RED first: 5 of 6 tests in `tests/test_hunter_curated_tools.py` failing; then green): `hunter_tools` gains `read_definition`, `entry_points`, `flows`, `obligations`; `run_tool_hunter` gains `system_prompt`, `user_prompt`, `context_files`, `tools`, `max_chars` and `tags`, all defaulting to today's behaviour, and replays the provider's reasoning field on the next turn. Deviations: `read_definition` returns the definition text as well as its span, and includes preceding decorator lines, because a resolver that returns only line numbers forces another `read_file` and a handler's decorators are exactly what the obligations work showed to matter; the three list tools return identifiers and spans only. No caching: a stale view of a tree a round may have rewritten is worse than a reparse. -- Review round 1 (2026-09-06) APPROVED 1.1, 1.2, 1.4, 1.5; REJECTED 1.3. The reviewer ran every check in a detached worktree at the group-1 tip and confirmed RED-first by running each new test file against the pre-feature commit (37 failures there, all passing after), the three gate outputs byte-identical, and `pairs`/`benchmark` free of any learning import. - - **1.3 rejection, remediated RED-first**: `_body_digest` stripped every leading comment line, which in C and C++ swallows `#include` and `#define`, and in any language swallows an ordinary leading comment. Two files differing only in a buffer size on a `#define` hashed equal and were marked `identical_twin`, removing a real pair from every denominator. Measured before the fix: twelve file sides across six vendored pairs had directives inside the stripped prefix, including the Juliet stack-overflow pair whose whole point is that constant. The strip is now anchored to the grammar `harvest.provenance_header` writes: a leading comment counts as header only when it carries one of the eleven provenance keys followed by a colon, and the scan stops at the first line of code. Four regression tests pin both directions plus a corpus-wide assertion that no vendored excerpt loses a directive. The five twins, ten duplicate groups and nineteen straddling repositories are unchanged, so the committed hygiene artifact still holds. - - Suggestions applied: a mechanism claimed by two families now fails loud by name; the obligations tool's path-escape refusal and the endpoint's degrade-to-None on a provider error are now asserted; the TypeScript parse assertion is exact instead of a 25% allowance, which surfaced one real corpus defect (`lensops-kubernetesmanagement-fe4910` was harvested as a truncated destructuring parameter list and is not valid TypeScript on its own; it belongs to the 4.1 re-harvest). - - **Two findings escalated to the maintainer, both blocking task 3.8 rather than group 1.** (a) `untrusted_destination` contradicts the approved requirement sentence, which says "server-side request forgery": the family id is the key every published number carries, so `requirements.md` Req 1.1 and the brief's Decision 1 must be amended to the widened name, or the family renamed back and CWE-601's 34 rows re-homed. (b) The shipped taxonomy is flat, so `parent_child` is unreachable in production and the "partial credit for a parent-child relation" in Req 2.3 and 3.1 is inert at version 1; either declare real parents or record the inertness beside every published agreement number. - - Carry-forward: the nineteen straddling repositories cross the split by declaration, so `split_by_repository` cannot fix them and task 3.1's teacher rule inherits the leak until a maintainer re-declares those splits. The `preprocess_repository` cost of the three list tools should be measured in 2.5 before round zero. -- Task 2.1 (2026-09-06, RED first: 8 failing tests in `tests/test_learning_classify.py`; then green): `learning/classify.py` with three tiers, multi-label answers in taxonomy order, and `validate_family_labels` owning the taxonomy check the corpus loader deliberately does not do. -- Task 2.2 (2026-09-06, RED first: 5 failing tests in `tests/test_learning_classifier_report.py`; then green): `measure_classifier` and `write_review_queue`. Bug caught by the RED test: the classifier read the declared label it was being scored against, making agreement trivially 1.0; `classify_pair(..., use_declared=False)` is what the measurement uses. Measured on the 277 vendored pairs: tier one places 275 without a model call, 2 land in `unknown`, and the 176 memory-safety rows resolve to one family. `labeled` is 0 because no row carries a maintainer family label yet, and the report says so instead of inventing a score. -- Task 2.3 (2026-09-06, RED first: 8 failing tests in `tests/test_learning_scoring.py`; then green): `learning/scoring.py` `score_pair_family`. Detection is a family-tagged finding inside the labeled function; text is never read. Findings of other families or outside the function are counted separately and never called leaks; a fabricated family id is counted as noise. K runs, majority outcome, disagreeing runs recorded as flips. -- Task 2.4 (2026-09-06, RED first: 7 failing tests in `tests/test_learning_aggregation.py`; then green): `aggregate` and `sign_test`. Recall at a fixed false-positive ceiling is forfeited entirely when the leak rate exceeds it; the directional-bias index separates "flags everything" from "silent on everything", which Youden cannot; change against a baseline is a two-sided exact sign test over per-pair flips, not a difference of means. `PairFamilyScore` gained a `slice` field (additive, filled from the case) so metrics can be reported per slice. -- Task 2.5 (2026-09-06, RED first: 9 failing tests in `tests/test_learning_detectors.py`; then green): `learning/detectors.py` with `FamilyConfig`, `Region`, `load_family_configs`, `run_family_detector`, `run_region_detectors`, `write_default_configs`. Deviations: the parts live in one `family.toml` plus the markdown and JSON-lines files, because two TOML files with three keys between them was more surface than value and `version` had no home in the design's list; configurations live under the project's learning directory, not inside the package, because a round writes them. `CURATED_TOOLS` is derived from `HUNTER_TOOLS` so the closed set cannot drift. -- Task 2.6 (2026-09-06, RED first: 6 failing tests in `tests/test_learning_verifiers.py`; then green): the optional oracle callback on `verdict_from_result`, inserted after the inconclusive guards and before the exit-code branch so a call without an oracle is the old decision tree, plus `verifier_for`, `StaticVerifier`, `NoVerifier` and `apply_verification`. -- Task 2.7 (2026-09-06, RED first: 8 failing tests in `tests/test_learning_canaries.py`; then green): `learning/canaries.py` with per-family templates, `CanaryVerifier` and `confirm_proven`. Deviation: templates live in `learning/canaries.py` rather than the design's `learning/verifiers/` directory; the property that matters, being outside every proposer write root, holds and is tested. -- Task 2.8 (2026-09-06, RED first: 6 failing tests in `tests/test_learning_journal.py`; then green): `learning/journal.py` with `RoundRecord`, `Attribution`, `LearningJournal`, `Archive` and `summarize`. -- Review round 2 (2026-09-06) APPROVED 2.1, 2.2, 2.4, 2.5, 2.8; REJECTED 2.3, 2.6, 2.7. The reviewer ran every check in a detached worktree at the group tip, re-derived the RED phase per task against each task's parent commit (8/5/8/7/9/6/8/6 failures, the four recorded claims exact), killed 15 mutants against the new tests, and hand-verified `sign_test` against the closed form for every (better, worse) with n <= 20. - - **2.3 rejection, remediated RED-first**: two of Req 4.1's five unscorable reasons had no code path, so a row whose language could not be parsed was reported as `unresolved_label`, a corpus labelling defect, when the cause was a tooling gap. The design's `unscorable_reason(case, *, parse_ok, ranges)` is reinstated as a public function, `score_pair_family` takes `parse_ok` and `entry_points`, and `missing_context` names a handler that arrived without the registration that makes it reachable, for the families whose bug shape is a missing check. - - **2.6 rejection, remediated RED-first**: `apply_verification` wrote `evidence_level="proven"`, which is not a member of the project's `EvidenceLevel`, so `verify_finding`, `is_report_verified` and `verify_judge` all raise `ValueError` on it. Requirements put the evidence ladder out of scope and the design's data contract specifies only the `verifier:` tag. Nothing in production called it yet, so the suite stayed green while task 3.4's wiring would have broken a scan. `apply_verification` now writes only the tag; `evidence_level_for` and `raise_to` are the single explicit bridge onto the ladder, and a test round-trips a proven claim through the project's own helpers. - - **2.7 rejection, remediated RED-first**: the injection canary opened its own in-memory database, inserted the token there and called the handler without ever giving it that connection, so the oracle could never fire; the fake sandbox runner returned the token regardless of the snippet, which is why no test noticed. It is now an execution canary whose marker only an interpreter could write, and each template is exercised end to end by running the rendered snippet against a vulnerable fixture and its fixed twin. That test immediately caught a second flaw: a fixed twin that echoes its input printed the oracle token and satisfied the oracle with no vulnerability at all, so an injected payload now carries a value unrelated to the token the oracle looks for. Recorded scope: the injection canary confirms command, code and template execution; a query-language injection whose payload never reaches an interpreter stays a suspicion. - - Suggestions applied: a configuration with an empty tool list is refused by name instead of producing a detector that silently reports nothing; the classifier abstains when its no-adapter fallback call also fails; the archive docstring states that the earliest winning version keeps its claim to a pair. - - Task-plan updates from the review: 2.2 and 2.4 claim the two criteria the maintainer added (2.6 and 3.7); 3.6 claims Req 6.2 with an observable that every non-target family directory is byte-identical across a round; 3.7 claims Req 3.5's K floor, 9.6's pair identities and 11.3's trajectories. - - **Escalated to the maintainer**: Req 6.1 lists "a skills set" among a detector configuration's parts, and neither the approved design nor the code has one, although the repository ships a skill router. Either Req 6.1 drops the skills set or a task wires it; the implementation followed the approved design, so this is a spec-internal conflict rather than a deviation. - - Carry-forward: the nineteen straddling repositories still cross the split by declaration and cap every holdout number these tasks produce; `unknown` serves as both the classifier's abstention and the generalist detector, which deserves a sentence in the design before a round ever targets it. -- Task 4.1 (2026-09-06, RED first: 21 of 23 failing tests in `tests/test_pair_registration_context.py`; then green, 26 tests). Measured before the change: **8 of the 10 vendored access-control rows had no named handler at all**, so every one of them would have scored `missing_context` forever rather than as a detector miss. After the re-harvest 9 of 10 are named and the tenth states its reason. What the repair needed, in order of how much it cost: - - **Class registration.** A flask-restx handler is a method; `@ns.route('/profile')` sits on the class. `extract_handler_context` now carries the enclosing class declaration and its decorators, so the excerpt is both reachable and parsable — a bare indented `def get` is neither. - - **Multi-line decorators and commented-out guards.** `@ns.doc(...)` wraps over four lines, and stopping at a continuation dropped the `@ns.route(...)` above it — the registration itself. `#@token_required` above a handler is not noise; in the two ThreatByte rows it is the entire bug, and the evidence field says so. `_decorator_start` now walks both. - - **Cross-file registration (design amendment, recorded in `design.md`).** connexion registers VAmPI's handlers by `operationId` in `openapi_specs/openapi3.yml`; Overleaf registers `exportProject` in `router.mjs`. No single-file excerpt can contain that, so a recipe may name `context = [...]`, the harvester writes the stanza per side under `/context//`, the catalog carries `[[pair.context]]`, and `_materialize_side` lays the documents out beside the excerpt at their repository-relative paths. `mapping._route_manifest_entry_points` reads `operationId` plus the operation's `security` block; `_js_statements` joins a registration that spans lines, because reading `webRouter.post(` alone finds neither the handler nor the middleware that guards it. - - **Anchors.** Real-Vuln-Benchmark trap rows take their fixed side from a different, correctly guarded handler of the same snapshot, so `handler_context` honours `line`/`fix_line` and a new `fix_function` gives the fixed side *its* registration instead of the vulnerable side's. - - **`innermost_function_bounds`** for brace languages, used only by this mode: `function_bounds_at` returns the outermost top-level group, which for a handler declared inside a React component is the whole 6000-line component. `hunk` and `enclosing` rows keep the bounds they were harvested with. - - **Fetch-then-write.** `harvest_recipe` fetches every blob before the first write, and the driver takes `--report` so a failed fetch leaves the reviewed excerpt exactly as it was and says why. A half-written re-harvest is indistinguishable from a real upstream change. - - Bugs the re-harvest exposed and fixed on the way: `.mjs` and `.cjs` were not in `LANGUAGE_BY_EXTENSION`, so the one pair vendored as an ES module had **no file target at all** — no entry point, no finding, a permanent miss; and `_python_route_access` matched auth words anywhere in a decorator, so `@ns.doc(description='... without proper authorization checks')` read the unguarded ThreatByte endpoint as `authenticated`, silencing every obligation on it. The guard is now the decorator's name, never its prose, and `@token_required` — the commonest Flask guard — is recognised at all. - - Corpus corrections: `lensops-kubernetesmanagement-fe4910` was labeled `setErrorMessage`, a React state setter, because the hunk landed in the wrong bounds; it is now `loadNamespaces` and the pair diff is exactly the fix. Its bug is a browser-side call omitting the header its siblings carry, not a missing guard on an inbound handler, so it carries `known_limit = "client_side_caller"` and leaves the denominators with a stated reason. - - Three existing tests encoded the old corpus and now encode the new one: the TypeScript parse pin (the lensops excerpt parses now), the classifier report (`labeled` 0 -> 4, the four seeded rows; the six `title`-tier agent rows carry a family but are not labels a maintainer stands behind), and the hygiene report ordering. - - **New honesty finding, recorded in the hygiene artifact as `cross_split_duplicates`: nine of the ten duplicate groups sit on both sides of the split.** Two forks of one project (`openniw-jobs-0fc947` train, `openniw-jobs-b74de3` holdout) and eight CVEs harvested twice under different names are byte-identical, so the train row teaches the holdout row exactly. `split_by_repository` cannot see it — the duplicates are different repositories. `duplicate_groups` is now ordered by name so the committed report does not reshuffle when one excerpt is re-harvested. - - Pre-existing drift the regeneration corrected, unrelated to this task: `catalog.toml` in git disagreed with a fresh `catalog_gen` run at the group-3 tip on two derived labels (`origin` -> `createApplication`, `normalizeDomain` -> `getGoogleProvider`). Verified in a detached worktree at `7a33756` before touching anything. -- Group 3 (2026-09-06). Tasks 3.1-3.9 implemented in `86ed54c`, `fcda1ee`, `e8e6baf`, `127dbef`, `ad7c98e`, `346cc70`, `16d9bc2`, `b955251`, `b8ad42c`, `7a33756`. RED counts, derived by the reviewer against each task's parent commit: 3.1 4/6, 3.2 6/6, 3.3 6/6, 3.4 3/5, 3.5 10/10, 3.6 12/12, 3.7 9/9, 3.8 7/7, 3.9 7/7. Deviations from the design, recorded here because they were not written down at the time: `learning.split` duck-types `PairCase` through `getattr` rather than importing it; `pairs.py` reaches `learning` through function-local imports rather than the design's injected callable; `run_baseline` takes a scan factory where the design named a client. One flaky test found and fixed in `346cc70` (`_is_vulnerable_side` keyed on `"-v" in str(root)`, which random temporary directory names matched about once in eight; harness noise recorded as detector noise would inflate every family's regression budget). -- Review round 3 (2026-09-06) APPROVED 3.4; REJECTED 3.1, 3.2, 3.3, 3.5, 3.6, 3.7, 3.8, 3.9. The reviewer ran every check in detached worktrees at the group tip against `71efc9c`, derived the RED phase per task, mutation-tested `decide`, the teacher rule, the snapshot restore and the round's revert paths (7 mutants on the acceptance rule, 6 killed and 1 equivalent), ran the three flakiness-prone files 61 times with varied `PYTHONHASHSEED` without a failure, and confirmed the three gate outputs byte-identical and the extra-free matrix unchanged. Every rejection was remediated RED-first; the numbers each defect was hiding are recorded below because they are the point of the spec. - - **3.2, the fixed side was judged against the vulnerable side's line ranges.** A fix that adds a guard above the handler moves its first line, and the reviewer measured that **191 of 227 pairs with a resolvable labeled span (84%) have different spans on the two sides**. A finding inside the fixed handler was therefore read as outside it: a leak recorded as silence, always in the direction that flatters Youden. `score_pair_family` now takes `fixed_ranges`; the fixed side's parse was already being computed and thrown away. - - **3.2 and 3.9, K runs was declared and never threaded.** `_evaluate_hunter_pair(..., runs=1)` had one call site that passed nothing, `evaluate_catalog` had no parameter at all, and `pairs --k-runs` was parsed and discarded; the only test grepped `--help`. K now flows `pairs --k-runs` -> `evaluate_catalog` -> `evaluate_pair` -> `_evaluate_hunter_pair`, defaults to three and refuses less, and the test counts scans per side. - - **3.2, the generalist could never be scored.** `pairs --hunter` fed the class-aware scorer findings with no `family:` tag, so it reported recall 0 by construction. `run_tool_hunter` now reads an optional `family` field into the tag (the design said it would) and `make_hunter_scan(taxonomy=...)` asks for one from the closed list. A generalist that names no family is unanswerable; one that names a wrong family is counted as noise, which is the honest treatment. - - **3.3 and 3.9, round zero measured every family with the generalist.** `run_baseline` took one `scan`, and the CLI handed it the `unknown` configuration; since `related(X, "unknown")` is `lateral` by the taxonomy's own decision that abstention earns no credit, **every family except `unknown` baselined at a structural zero** and every noise floor with it. `run_baseline` now takes a `scan_factory` and runs each family's own configuration, reports per slice as well as per family, and refuses fewer than three runs. This is the decision the group-2 review asked for on `unknown` serving as both abstention and generalist: it is the generalist for *scanning*, and it is never credited as a family for *scoring*. - - **3.5, the meta-agent proposer was a stub** that returned None after confirming the extra was present, so `ousast learning round` could never propose anything at any time. It now composes `MetaAgent.evolve` behind `harness_ext` against a scratch **copy** of the family directory — the live tree is never touched, because the round snapshots only after a proposal exists — and builds the proposal from the diff of that copy, so the one-file rule and the lever whitelist are enforced on what the agent did rather than on what it claimed. `learning round --proposals FILE` runs a recorded sequence offline, which is what makes the command exercisable without the extra at all. - - **3.6 and 3.1, the holdout refusal existed only as a vocabulary word.** `journal.py` documented `refused_holdout` and no code path could produce it; the covering test asserted `outcome == "refused_holdout" or outcome == "rejected"`, which passes whichever way the code behaves. The round now runs `refuse_if_holdout` over every corpus pair the proposal names — in its predictions and in the file it writes — and journals `refused_holdout` with the pair names. A memory entry that quotes a held-out pair looks exactly like a legitimate hard negative until the holdout number is read as if it meant something. - - **3.7, the sweep counted absolute failures, not negative flips.** Req 9.5 compares against the pre-change state; the code counted every pair of another family that was not `pair_correct` and compared that to a budget derived from the run-to-run flip rate. On the real corpus, where most pairs fail at round zero, that rejects **every** round for damage it did not do — a harness that then reports it cannot improve. Unscorable rows voted too, against Req 4.1. The sweep is now scored before and after the change, `_negative_flips` counts only pairs that were right and are not, unscorable rows leave the count, and attribution is derived from the same comparison instead of from absolute failures. - - **3.7, cost was metered once**, after the minibatch, the holdout and the whole cross-family sweep had all been paid for; Req 9.8 says stop, revert and record. The cap is now read after every stage. A stage that raises is journalled as `reverted`/`stage_failed` with its round directory before the exception leaves, because an exception that loses the record also loses the money the round already spent. A rejected round's version is archived: Req 9.7 exists so a configuration that lost overall but is the only one that ever won some pair is not forgotten. - - **3.8, the corrected lever number was published beside the earlier one, not in place of it** (Req 5.3), and the test named for that property never checked it. `publish` now rewrites the superseded claim to point at the regenerated section, idempotently, and the test asserts the old sentence is gone. Run against the real roadmap: the 2026-09-05 claim "human holdout profile pair_correct 0 -> 6 of 17, Youden 0 -> +0.353" now says it was measured with holdout pairs teaching and names the artifact that corrects it. - - **3.1, refusals were `continue` statements.** `Refusal.degradation()` existed and nothing called it. The mechanism lever now records every candidate it declined with the holdout pairs that taught it, leave-one-out records the pairs it kept out of the teaching set, and `improve` carries both into the round outcome. `semantic.seed` reported `split:train` for a train-split identical twin, which is simply untrue; an unscorable row now reports its own reason. `evaluate_profiles` and `evaluate_mechanism_profiles` are deliberately split-blind and now say so in their docstrings: they measure a ruleset and never learn from a pair, so a filter there would only shrink the evidence. - - Test-quality repairs the review asked for, each of which changed a number: two split tests were measuring an `x = 1` fixture that teaches nothing whatever the rule says, so both arms now use a pair that really carries a shape and the lever test asserts the same shape *is* admitted when a train pair taught it; the two CLI scoreboard tests asserted against output that was all zeros, and now use a side-aware detector stub (the shipped `scripted` stub answers twice, then goes quiet, and puts its finding outside the labeled function — three separate ways to make a family number zero); `ScriptedChatClient` gained an opt-in `cycle` so a stub cannot silently run out of answers mid-run; the sandbox-candidate assertion in `tests/test_learning_scan.py` compared verdict paths against a prefix no implementation would produce and now compares finding ids; the manifest's classifier block reported `measure_classifier([])`, all zeros, and now reports what this scan actually routed. - - Carry-forward, unchanged and still capping every holdout number: the nineteen straddling repositories, and — found during task 4.1 — **nine of the ten duplicate groups sit on both sides of the split**, which the repository-level split cannot see because the duplicates are different repositories. -- Task 4.2 (2026-09-06, RED first: 2 failing tests in `tests/test_fixture_difficulty.py`, and the check's own arm — `test_the_check_catches_a_fixture_that_is_easier_on_one_axis` — fails if the fixture hardening is reverted). Two parts: - - **The five twins declare their limit.** A content hash is evidence; the catalog row is the maintainer standing behind it, and without one a re-harvest that changes a byte turns a known-unscorable row back into a scored one with no review. A declared limit that names one of the vocabulary's own reasons keeps that spelling, so `identical_twin` declared and `identical_twin` computed stay one fact instead of splitting across two buckets in every aggregation. `semantic.seed._skip_reason` was reordered for the same reason: an unscorable row reports `unscorable:` before either the known-limit or the split branch can claim it. Consequence, recorded because it moves a published number: leave-one-out over vibe-py now runs on **30 rows, not 35** — a pair whose two sides are the same code cannot be held out against itself. - - **The fixture-difficulty check** (`learning/difficulty.py`), on three axes the corpus really varies on: binding hops from the identity source to the constrained use, registration distance (0 decorator on the handler, 1 statement elsewhere in the file, 2 another document), and guard distance (0 decorator, 1 call in the body, 2 nothing visible). The comparison is per axis over the whole set, because a suite covers a bug shape with several handlers between them. **Measured first, and it failed: the shared obligations fixture reached (1, 0, 2) where the vibe-py absence rows reach (5, 2, 2).** Every shared handler carried its own route decorator, so nothing in the suite exercised a handler registered from another document and no chain was longer than one binding — the checker was passing on a simpler world than the one it meets. `from_spec` was added to the fixture: the connexion shape those rows really have, a token validated by a call rather than a decorator and five bindings between the token and the constraint. The verdict is committed as `benchmarks/measurements/2026-09-06-fixture-difficulty.json` with both reaches and the history of the first measurement. One existing expectation moved with it (`from_spec` reads the user row and then the book: two real operations). -- Task 4.3 (2026-09-06, RED first: 10 failing tests in `tests/test_learning_cutoff.py`; then green, 11). The catalog generator fills `fix_date` from the recipe at the precision the recipe has: a full ISO day when one was recorded, otherwise the year. Every harvested recipe carries a year and almost none carries a day, and writing `-01-01` would turn a year into a claim about a month. A row therefore counts as post-cutoff only when its **earliest possible** date is after the cutoff, so a coarse date can under-claim the clean slice but never over-claim it — and over-claiming is the direction that would turn contamination into a result. `PairFamilyScore` carries the date, `aggregate(cutoff=...)` reports the same numbers over the post-cutoff rows beside the whole number and never instead of it, undated rows are counted on their own, and `publish` prints the slice with the cutoff it used. Without a configured cutoff every scoreboard says so rather than inventing one; `[learning].cutoff_date` has no default because guessing a model's training cutoff and publishing a contamination number from the guess would be worse than reporting none. Corpus dates as harvested: vibe-py all 2026, agent-vfc 67 of 69 in 2026, vfc spread 2014-2026, vfc-js 2014-2021. -- Task 5.1 (2026-09-06, RED first: 3 of 9 tests in `tests/test_degradation_matrix.py`; then green, 8 after one test was deleted rather than kept). The three gate outputs are captured at `71efc9c` — the commit before the first line of this feature — as `benchmarks/measurements/2026-09-06-gate-baseline.txt`, and a test reproduces them byte for byte on every run, in both matrices. Until now byte-identity was something a reviewer checked by hand against a worktree; it is now mechanical and runs in CI, which is what makes it a gate rather than a habit. Six absences are each exercised once with the reason they record: HarnessX (`harnessx_unavailable`), the chat endpoint (`learning_endpoint_unavailable`, and the run still writes its artifact), the family configurations (`learning_configs_unavailable`), the network for pointer pairs (`pointer_pair_skipped`), the semantic extra (`language_unsupported` rather than a guess), and the sandbox (`sandbox_unavailable`, so a claim nothing could run is not confused with a claim that failed). One test written for this task was **deleted before the commit**: it asserted that this file contained a `without_` name per capability, which measures the file's own naming and nothing about the code. -- Review round 4 (2026-09-06) APPROVED 4.1, 4.2, 4.3; REJECTED the round-3 remediation on three counts. The reviewer re-derived the RED phase for group 4 (24 of 26 for 4.1, 3 for 4.2, 11 of 11 for 4.3 — **every count in the notes above was understated**, corrected here), killed 18 of 19 mutants, verified the 4.1 observable independently (2 of 10 access-control rows named at `7a33756`, 9 of 10 at `3e71e9e`, the tenth stating its reason), confirmed `_python_route_access` changed no access level anywhere in the corpus and `.mjs` changed exactly one snapshot, and reproduced the fixture-difficulty artifact number for number including its failing arm. Remediated RED-first: - - **The meta-agent proposer still could not propose.** `MetaAgent(inner_model=...)` takes a `ModelConfig`; it was handed a model name, a `# type: ignore` hid that from mypy, and a bare `except Exception` turned the resulting `AttributeError` into the degradation reason `meta_agent_failed` — which reads exactly like the model declining. Every other HarnessX composition site in the repository builds the config properly. `_meta_agent_class` and `_model_config` are now seams a test can pin without the extra, and a `TypeError`/`AttributeError`/`ImportError` from the call is re-raised: "we called the agent wrong" is not "the agent had nothing to propose". - - **`ousast pairs --hunter` crashed in its documented form.** The K floor landed with `--k-runs` still defaulting to 1, so the flag's absence raised `ValueError` from inside `evaluate_catalog`. The default is `None` (three), and a breach exits naming the flag. The covering test always passed `--k-runs 4`, which is why nothing noticed. - - **28 rows resolved a span on the vulnerable side and none on the fixed side** (12% of the corpus), so every fixed-side finding read as outside the labeled function and the pair scored as silence whatever the detector said — the same flattering direction as the range defect, one-seventh the size. `fix_function` reached only the harvester; it now reaches the catalog and `PairCase`, `_spans` uses it for the fixed side, and the catalog generator derives it from the fixed excerpt itself for the 25 vibe-py trap rows that never declared one. A row that still resolves no fixed-side span reports `unresolved_label:fixed` instead of scoring silent. Measured after: **0 rows**. - - Suggestions applied: one spelling per corpus fact everywhere (`PairCase.unscorable` verbatim, no invented prefix); leave-one-out records non-teachers withheld for tier or an unscorable reason, not only holdout refusals; the hygiene test asserts the declared twins still hash equal, so a declaration that stopped being true would not be invisible; `is_post_cutoff` refuses a cutoff coarser than a day, which was the one over-claim path that existed, and month precision is pinned; `UNSCORABLE_REASONS` covers the whole vocabulary rather than `identical_twin` alone; the zero-run score carries its date; `_pairs_named_by` filters by substring before the regex. -- Carry-forward, measured 2026-09-06 after group 4 (`benchmarks/measurements/2026-09-06-trajectory-divergence.json`): **the K-run numbers this spec publishes are ensemble votes, not repeated measurements.** Across 120 runs of the injection family at temperature 0 with thinking disabled there was not one repeated tool-call trajectory, and outcome disagreement tracks trajectory divergence (30.9% at low similarity, 4.2% at mid). Consequences for anyone reading these numbers: Req 3.5's K is an ensemble over distinct search procedures; Req 8.2's noise floor measures how often an unconstrained search reaches the sink rather than the detector's uncertainty, so Req 9.5's regression budget grows with the search's own wandering; and the instability is entirely one-sided — every pair produced a single finding set on the fixed side, so specificity is solid and recall carries the caveat. `.kiro/specs/detector-optimizer/brief.md` proposes the repair, and it is a program change rather than a prompt change. -- Task 5.3 (2026-09-07, five rounds on `injection`, $2.17, `benchmarks/measurements/2026-09-07-evolve-rounds-injection.json`). The task was amended to target injection because access_control has one train teacher. - - | round | lever | outcome | train | p | holdout | p | distinguishable from noise | - |---|---|---|---|---|---|---|---| - | 1 | memory | rejected `train_regressed` | 0+/3− | 0.250 | 1+/4− | 0.375 | no | - | 2 | checklist | rejected `train_regressed` | 0+/1− | 1.000 | 1+/1− | 1.000 | no | - | 3 | prompt | **accepted `improved` → v1** | 2+/0− | 0.500 | 2+/1− | 1.000 | no | - | 4 | meta-agent | rejected `no_proposal` | — | — | — | — | never ran | - | 5 | counterexample | rejected `train_regressed` | 0+/1− | 1.000 | 1+/3− | 0.625 | no | - - **Nothing the loop decided carried evidence, in either direction — five rounds, zero reliable decisions.** Round 2 rejected a change because **one pair of eight** moved, at p = 1.000 — the maximum the sign test can return. Round 3 accepted one because two of eight improved and none regressed, at p = 0.500, and that acceptance is now versioned as v1 and will be built on. On a family whose measured floor says **11 of 20 pairs disagree with themselves between runs of the unchanged detector**, both readings are what changing nothing looks like. - - I overstated this once and correct it here: after rounds 1 and 2 I wrote that the rule was "close to structurally incapable of accepting anything". Round 3 accepted. The accurate statement is symmetric and worse — the rule accepts on noise as readily as it rejects on it. - - - **Escalation to the maintainer, not applied here.** Req 9.5's threshold rule (reject if either side regresses, accept if one improves and neither regresses, tie rejects) should be replaced by an acceptance that survives the measured variance. `Archive` already holds the per-pair Pareto frontier and `acceptance.reliability` already computes the test; neither decides anything. The honest caveat is that at a minibatch of 8 and a holdout of 10 a reliability-gated rule would accept almost nothing either: **the corpus is the binding constraint, not the rule**, and no acceptance rule fixes a train split of twelve. - - **The v1 configuration rests on p = 0.500** and should probably be reverted before any further round builds on it. Left in place, and flagged, because reverting an accepted round is the maintainer's call. - - Round 1's rejection was correct but for my error, not the loop's insight: the `memory` lever writes `hard_negatives.jsonl`, which the detector renders as *"Known non-issues, do not report these"*, and I fed it the four vulnerability shapes the train split teaches. Round 5 re-ran the same content through `counterexamples` (*"Worked examples"*), which is the lever I meant — and was rejected because **one pair of eight** moved, at p = 1.000. **The experiment I designed to give the demo lever a fair test could not give it one.** The rule cannot distinguish a lever that helps from a lever that does nothing, so correcting my authoring error changed the reason for the verdict and not the verdict. - - What did work: the cross-family sweep caught a genuine negative flip in `access_control` in round 2 — a real regression found by the before/after comparison that replaced the absolute-failure count, which under the old rule would have rejected rounds for families that were already failing. - -- **The finding that outranks the rounds (maintainer, 2026-09-07: "the error is not the model, the error is the elsewhere").** Eighteen defects were found in this session's measurement and round work. Every one was in this repository; none was the model's. The list is kept because it should change how the next bad number is triaged: the prose parser that discarded findings the model had already reached; the CLI that never read the cost meter it published; the baseline that lost everything on an exception, and again on a timeout; two keys that collapsed a slice into another, in two different modules; the pattern ruleset at a 12.1% ceiling; a harvester that truncated an excerpt at both ends; a family shape that could not see a permissive default inside a call argument; a lever wired to the file that says *do not report these*; a provider seam hard-coded to a vendor with no key; a proposer reason the journal threw away; a retry that did not cover a body which stopped mid-read; an exception path wrapped around half a round; log-probabilities read from the wrong object; `.mjs` absent from the language table; and an access-level check that matched prose inside a decorator. Three of them were the same shape — fixing an instance and missing the class. **The default assumption for an unexplained number is that we broke it.** `deepseek-v4-pro` scoring 46.7% against flash's 83.3% is unexplained and is therefore, by this prior, ours until shown otherwise. diff --git a/.kiro/specs/openrouter-sast-harness/assessment.md b/.kiro/specs/openrouter-sast-harness/assessment.md deleted file mode 100644 index d4e8921..0000000 --- a/.kiro/specs/openrouter-sast-harness/assessment.md +++ /dev/null @@ -1,325 +0,0 @@ -# Assessment: OpenUltraSAST Kickoff - -## Executive Assessment - -OpenUltraSAST should be built as a harness runtime specialized for security analysis, not as a direct Clearwing clone. The strongest design is to treat Clearwing's sourcehunt pipeline as a proven reference flow, HarnessX as the runtime abstraction, OpenUltraCode as the verification and fusion discipline, OpenRouter as the provider plane, and Trail of Bits skills as targeted domain expertise. - -The minimum viable system is not deep fuzzing. The minimum viable system is an auditable quick scan that can rank files, generate structured findings, eliminate obvious false positives, enforce evidence levels, and emit reports. Deep Docker-backed validation should come after the evidence model, trace model, false-positive loop, and report model are stable. - -## Why This Project Is Viable - -Clearwing demonstrates that a SAST + LLM hybrid can be useful when it avoids treating model output as truth. Its best transferable ideas are: - -- Static preprocessing before LLM analysis. -- File ranking by surface, influence, and reachability. -- Tiered hunter budgets instead of equal attention for every file. -- Independent adversarial verification. -- Evidence ladder that separates suspicion from crash reproduction and patch validation. -- Docker isolation for builds, fuzzing, PoCs, and patches. -- SARIF, Markdown, and JSON artifacts. - -The most important product differentiator should be false-positive elimination. A scanner that finds many plausible issues but cannot separate evidence-backed findings from model hallucinations will not be trusted. Ranking, retrieval, verification, and HarnessX-style adjustment should all be optimized around reducing false positives without hiding the audit trail. - -HarnessX provides the missing generalization: the scanner should be a composed harness with typed processors and traces, not a fixed pipeline that becomes hard to evolve. This matters because SAST workflows vary heavily by language, build system, vulnerability class, and available evidence. - -OpenUltraCode provides the missing discipline: high-risk findings and proposed fixes need bounded plans, adversarial review, reconciliation, explicit model disclosure, and verification gates. This is directly applicable to false-positive control and to avoiding fixes that look plausible but introduce regressions. - -Trail of Bits skills provide mapping discipline as much as they provide tools. CodeQL thinking maps interprocedural flows, Semgrep maps repeatable patterns and variants, differential review maps changed attack surface, and sharp-edge review maps APIs and defaults that make misuse likely. OpenUltraSAST should convert those disciplines into structured tasks and evidence, not only prompt inspiration. - -## Recommended Initial Product Shape - -The first release should support this workflow: - -```text -ousast scan /path/to/repo --mode quick -``` - -The first realistic quality milestone should add benchmark-grounded evaluation before claiming language coverage: - -```text -ousast benchmark benchmarks/javascript-node.toml --mode standard -ousast benchmark benchmarks/python-web.toml --mode standard -ousast benchmark benchmarks/java-web.toml --mode standard -ousast benchmark benchmarks/c-cpp.toml --mode quick -``` - -It should produce: - -```text -.openultrasast/runs//manifest.json -.openultrasast/runs//preprocess/file_targets.json -.openultrasast/runs//rank/ranking.json -.openultrasast/runs//findings.json -.openultrasast/runs//report.md -``` - -The key product promise should be: - -```text -OpenUltraSAST tells you where security review should start, what it suspects, what evidence exists, what evidence is missing, and which findings have survived independent verification. -``` - -For coder education, the complementary promise should be: - -```text -OpenUltraSAST explains why the pattern is dangerous, what evidence proves or weakens the finding, how to fix it safely, and how to avoid recreating the same bug class. -``` - -## Core Architecture Recommendation - -Use a Python implementation initially. Clearwing and HarnessX are both Python, Docker and SARIF libraries are mature, and Trail of Bits tooling integration will be simpler. OpenCode integration can happen through MCP and command files rather than requiring the core scanner to be TypeScript. - -Recommended modules: - -```text -openultrasast/config.py -openultrasast/cli.py -openultrasast/models.py -openultrasast/provider/openrouter.py -openultrasast/preprocess/ -openultrasast/index/ -openultrasast/rank/ -openultrasast/harness/ -openultrasast/hunters/ -openultrasast/verify/ -openultrasast/fusion/ -openultrasast/sandbox/ -openultrasast/reports/ -openultrasast/mcp/ -``` - -## Build vs. Reuse Decisions - -| Area | Recommendation | Reason | -| --- | --- | --- | -| Harness runtime | Build minimal HarnessX-inspired core first | Direct HarnessX dependency may add unstable surface before needs are proven | -| OpenRouter chat | Build direct OpenAI-compatible client | Need explicit model IDs, cost metadata, and role controls | -| OpenRouter embeddings | Build direct client | Embeddings are central and should not be hidden behind chat abstractions | -| Vector store | Choose one local default | Avoid abstracting before retrieval needs are proven | -| Docker sandbox | Build focused wrapper | Security defaults are project-specific | -| Trail of Bits skills | Index descriptors, mapping tasks, and snippets | Avoid prompt bloat while preserving CodeQL/Semgrep/differential/sharp-edge discipline | -| Clearwing | Reference only | Independence is a project requirement | - -## Configuration Shape - -Model names and the vector store should be selected before implementation. The config shape should look like this: - -```toml -[models] -ranker = "" -hunter = "" -verifier = "" -patcher = "" - -[embeddings] -model = "" -store = "" - -[sandbox] -network = false -workspace_readonly = true -memory_mb = 2048 -timeout_seconds = 300 -pids_limit = 512 - -[dynamic] -enabled = false -network_scope = [] - -[evidence] -minimum_report_verified = "static_corroboration" -minimum_exploit = "crash_reproduced" -minimum_patch = "root_cause_explained" -``` - -The actual model names should be explicit before implementation. Avoid baking in placeholder defaults that silently pick a moving target. - -## Vector Store Decision - -Choosing the vector store is a core design decision, not plumbing. It directly affects ranking quality and false-positive rates because poor retrieval contaminates hunter and verifier context. - -Selection criteria: - -- Strong metadata filters for path, language, symbol, repository snapshot, evidence level, and vulnerability class. -- Incremental indexing for changed files. -- Local persistence that works inside opencode workflows. -- Deterministic export/import for reproducible scan artifacts. -- Acceptable query latency on medium repositories. -- Clean storage of OpenRouter embedding model metadata. -- Ability to combine vector retrieval with lexical or structural filters. - -The first milestone should include a short bakeoff on fixtures before locking the default store. - -## Most Important Engineering Constraint - -Evidence transitions must be enforced in code. A model can recommend a transition, but the harness should decide whether the required artifact exists. - -Examples: - -- `suspicion -> static_corroboration`: requires static evidence, independent verifier agreement, or a matching analyzer finding. -- `static_corroboration -> crash_reproduced`: requires stored crash/reproducer artifact. -- `crash_reproduced -> root_cause_explained`: requires verifier explanation tying crash to source and impact. -- `root_cause_explained -> exploit_demonstrated`: requires exploit or trigger artifact beyond accidental crash. -- `root_cause_explained -> patch_validated`: requires patch diff and passing validation command. - -False-positive elimination should be handled the same way: a model can recommend demotion or suppression, but the harness should require a scoped reason and counter-evidence. - -Benchmark grounding should be handled with the same discipline. A scanner that passes tiny fixtures but misses NodeGoat, PyGoat, OWASP Benchmark/WebGoat, Juliet, or small sanitizer-backed parser targets is not a realistic finder. Language support should be claimed only after benchmark runs show useful recall and auditable false-positive behavior on realistic vulnerable projects. - -False-positive reasons should include: - -- No attacker-controlled path. -- Code is unreachable in supported configuration. -- Static rule matched a safe wrapper or sanitizer. -- Model confused similar symbols or files. -- Finding duplicates an existing issue. -- Impact is not security-relevant. -- Dynamic probe or test disproved triggerability. - -## OpenRouter Design Needs - -OpenRouter should be treated as two capabilities: - -1. Chat/completion routing for reasoning roles. -2. Embedding endpoint for retrieval. - -Every model call should record: - -```text -role -model_id -provider metadata -prompt hash -response hash -cost estimate -temperature and parameters -fallback/degradation status -``` - -The project should reject ambiguous model routing for reproducible scans. If a role uses auto routing, the manifest must record the actual model selected by OpenRouter if available. - -## Docker Design Needs - -Docker is not just convenience. It is the trust boundary between arbitrary target repositories and the user's host. - -Minimum safety profile: - -- No hidden network in analysis containers. -- Read-only source mount. -- Writable scratch only. -- Timeouts around every command. -- Memory, CPU, and pids limits. -- Dropped capabilities by default. -- Separate writable copy for patch validation. - -Deep mode should not be implemented until this boundary exists. - -## Dynamic Analysis Strategy - -Network verification is not out of scope. It is just not the core idea of the harness. OpenUltraSAST should remain SAST-first, but it should use dynamic analysis when runtime evidence is the right way to confirm a finding. - -Useful dynamic checks include: - -- Starting a target service in a sandbox. -- Using netcat-style TCP probes against declared local ports. -- Running local HTTP or protocol smoke tests. -- Executing a bounded PoC against a sandboxed service. -- Using Clearwing-style network checks as validation patterns when a source finding has a runtime network surface. - -The rule is not "no network ever." The rule is "no hidden or open-ended network activity." Dynamic probes must be explicit, scoped, logged, time-bounded, and tied to evidence advancement. - -## Trail Of Bits Skill Strategy - -The skills are best used as a routing corpus and checklist library. The harness should select compact snippets by target profile. - -Good examples: - -- C parser with pointer arithmetic: `c-review`, `address-sanitizer`, `libfuzzer`, `harness-writing`. -- Python parser: `atheris`, `property-based-testing`. -- Crypto code: `constant-time-analysis`, `wycheproof`, `vector-forge`, `zeroize-audit`. -- SARIF flow: `sarif-parsing`, `semgrep`, `codeql`. -- Changed code or proposed fixes: `differential-review`, `graph-evolution`. -- Dangerous APIs, defaults, or configuration: `sharp-edges`, `insecure-defaults`. -- Smart contracts: use chain-specific scanners only when the language and framework match. - -Bad pattern: - -```text -Paste all security skill instructions into every hunter prompt. -``` - -That would increase cost, reduce precision, and mix irrelevant assumptions into reviews. - -## Fix Authoring And Audit Strategy - -UltraCode should define the patch lifecycle. A patch is not complete when a model emits a diff. A patch is complete only after a bounded plan, minimal implementation, adversarial review, reconciliation, and fresh verification. - -Fix stages: - -```text -verified finding -> bounded fix plan -> minimal patch -> sandbox validation -> adversarial fix audit -> reconciliation -> fresh verification -> patch artifact -``` - -Trail of Bits disciplines should audit the fix: - -- Differential review checks whether the patch changed more attack surface than intended. -- Semgrep checks whether the vulnerable pattern or variants remain. -- CodeQL checks whether taint/dataflow still reaches the sink when applicable. -- Sharp-edge review checks whether the fix leaves a misleading or misuse-prone API behind. - -Fusion should always be used to go deeper when the task requires it: risky fixes, contradictory evidence, difficult remediation tradeoffs, high-impact reports, unresolved verifier disagreement, or explicit high-assurance review. It should not be spent on routine low-risk patches when ordinary evidence is already sufficient. - -## Kiro Spec Completion Criteria - -The current spec is ready to drive implementation when these decisions are made: - -1. Language/runtime: Python is recommended. -2. Vector store: choose Chroma, LanceDB, Qdrant local, or SQLite vector extension. -3. Initial OpenRouter chat models by role. -4. Initial OpenRouter embedding model. -5. First fixture repository or synthetic vulnerable examples. -6. Whether to depend directly on HarnessX or implement a minimal compatible harness core. - -## Recommended MVP Cut - -MVP should include: - -- Local path scan. -- File enumeration and static tags. -- OpenRouter ranker. -- Quick-mode hunter. -- Evidence ladder with enforced transitions. -- Independent verifier. -- JSON and Markdown reports. -- Manifest with model IDs and prompt hashes. - -MVP should exclude: - -- Deep fuzzing and broad dynamic analysis. -- Auto-patching. -- Fusion panels for routine MVP findings. -- Full MCP server. -- Multi-repository campaigns. -- Training bridge. - -MVP should still model the future fix lifecycle in the data model, because evidence, reports, and patch artifacts need stable IDs before patching exists. - -These exclusions keep the first milestone testable and prevent architecture from hardening around unverified assumptions. - -## Implementation Order Rationale - -Do not start with Docker or broad fusion. Those are expensive and only pay off after the data model and evidence model are correct. Start with the smallest useful loop: - -```text -preprocess -> rank -> quick finding -> verify -> report -``` - -Once this loop works on fixtures, add embeddings. Once retrieval is stable, add sandboxed commands. Once sandboxing is stable, add deep fuzzing. Add fusion as soon as tasks require deeper adjudication than the normal verifier can provide. - -## Immediate Next Questions - -1. Which OpenRouter chat models should be used for ranker, hunter, verifier, and patcher roles? -2. Which OpenRouter embedding model should be the default? -3. Which vector store should be selected for the first implementation? -4. Should the project start with synthetic fixtures or scan a real open-source target first? -5. Should the first implementation include MCP, or should MCP wait until the CLI and artifacts stabilize? diff --git a/.kiro/specs/openrouter-sast-harness/decisions.md b/.kiro/specs/openrouter-sast-harness/decisions.md deleted file mode 100644 index fd17d14..0000000 --- a/.kiro/specs/openrouter-sast-harness/decisions.md +++ /dev/null @@ -1,29 +0,0 @@ -# Decisions: OpenRouter SAST Harness - -## Approved Initial Slice - -The first implementation slice is approved for Phase 0 and Phase 1 only: - -- Python package skeleton with `ousast` CLI. -- Local configuration loading. -- Scan run directory creation under `.openultrasast/runs//`. -- Repository snapshot collection. -- Source file enumeration with basic ignore handling. -- Language, LOC, and static tag heuristics. -- `preprocess/file_targets.json` artifact. - -## Current Decisions - -| Decision | Value | Scope | -| --- | --- | --- | -| Runtime | Python 3.11+ | Initial implementation | -| Harness dependency | Minimal HarnessX-compatible core first | Avoid direct dependency until contracts are proven | -| Vector store | Deferred to Phase 4 bakeoff | Not needed for Phase 0/1 | -| OpenRouter chat models | Deferred until ranker implementation | Not needed for Phase 0/1 | -| OpenRouter embedding model | Deferred until embeddings implementation | Not needed for Phase 0/1 | -| Fixtures | Synthetic fixture repositories first | Keeps tests deterministic | -| MCP | Defer until CLI and artifacts stabilize | Phase 12 | - -## Rationale - -The first milestone should prove the local artifact contract before adding model calls, embeddings, Docker, MCP, or fusion. Later decisions must be recorded here before their implementation phases begin. diff --git a/.kiro/specs/openrouter-sast-harness/design.md b/.kiro/specs/openrouter-sast-harness/design.md deleted file mode 100644 index ba13159..0000000 --- a/.kiro/specs/openrouter-sast-harness/design.md +++ /dev/null @@ -1,762 +0,0 @@ -# Design: OpenRouter SAST Harness - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Architecture Summary - -OpenUltraSAST is a harness-oriented SAST system, not a monolithic scanner. The core runtime composes processors, model roles, retrieval, tools, sandbox policy, traces, verification gates, and reporting into a reproducible scan harness. - -The central quality objective is false-positive elimination. The harness should not merely find more possible bugs; it should repeatedly narrow suspicion into evidence-backed findings and learn from rejected claims. - -Conceptual lineage: - -- HarnessX: harness primitives, event hooks, processor contracts, trace-driven evolution. -- OpenUltraCode: bounded context, fix authoring discipline, adversarial review, reconciliation, fusion, verification evidence. -- Clearwing sourcehunt: preprocess, rank, tiered hunters, evidence ladder, adversarial verifier, patch oracle, variant loop, reports. -- Trail of Bits skills: domain-specific mapping, analysis, audit, and verification methods selected at the right stage, not embedded globally. - -## High-Level Flow - -```text -repo path - -> intake snapshot - -> preprocess and static tags - -> graph, SARIF ingest, and entry-point reachability mapping - -> embeddings and retrieval index - -> file/function ranking - -> tiered hunter pool - -> independent verifier - -> optional fusion adjudication - -> optional explain and guided fix workflow - -> optional reproducer/exploit triage - -> optional patch oracle - -> reports and artifacts -``` - -Benchmark runs wrap the same scan pipeline instead of using a separate detector path: - -```text -benchmark manifest - -> acquire or locate vulnerable project - -> run quick, standard, or deep scan - -> normalize OpenUltraSAST findings and optional external baselines - -> match expected vulnerabilities by CWE, file, function, sink, and evidence - -> record recall, precision where possible, misses, false-positive reasons, and runtime - -> feed misses and rejected findings into calibration artifacts -``` - -## Main Components - -### CLI - -Initial command surface: - -```text -ousast scan --mode quick|standard|deep --config openultrasast.toml -ousast benchmark --mode quick|standard|deep -ousast explain --scan --audience concise|learner|reviewer -ousast status -ousast findings -ousast report --format markdown|json|sarif -ousast patch --propose -ousast index --rebuild -``` - -The CLI should be thin. It loads config, creates a scan, starts the harness runtime, and writes artifacts. - -### MCP Server - -MCP should expose only stable project operations: - -```text -openultrasast.scan -openultrasast.status -openultrasast.findings -openultrasast.get_finding -openultrasast.evidence -openultrasast.artifacts -openultrasast.benchmark -openultrasast.explain -openultrasast.propose_patch -openultrasast.export_report -``` - -The MCP server must not expose arbitrary shell, Docker, or internal hunter tools. Those stay inside controlled harness stages. - -### Harness Runtime - -The runtime owns the scan lifecycle and event pipeline. It should follow the HarnessX separation: - -```text -ModelConfig: role -> OpenRouter model, parameters, budget, fallback policy -HarnessConfig: processors, tools, memory, sandbox, trace, control policy -ScanConfig: target path, mode, scope, excludes, report options -``` - -Required event hooks: - -```text -task_start -stage_start -before_model -after_model -before_tool -after_tool -stage_end -task_end -``` - -Each processor declares the event types it handles and the state fields it reads or writes. Strict contract mode fails the scan when processors violate schemas. Warn mode records degraded state. - -### OpenRouter Provider - -The provider layer should implement two explicit clients: - -```text -OpenRouterChatClient -OpenRouterEmbeddingClient -``` - -Chat roles: - -```text -ranker -hunter -verifier -fusion_panel_a -fusion_panel_b -fusion_decider -patcher -reporter -``` - -Provider rules: - -- No silent model substitution. -- Every model response records model ID, provider metadata when available, token counts, cost estimate, prompt hash, and response hash. -- Fallbacks must be configured per role and reported as degradations. -- Embedding model ID must be stored in the vector index metadata. - -### Retrieval Store - -The retrieval layer maintains separate namespaces: - -```text -repo_code -repo_docs -static_findings -mechanisms -trailofbits_skills -scan_traces -false_positive_learnings -``` - -Chunk metadata should include: - -```json -{ - "repo_id": "...", - "snapshot": "...", - "path": "src/parser.c", - "symbol": "parse_frame", - "language": "c", - "chunk_type": "function", - "start_line": 120, - "end_line": 220, - "tags": ["parser", "memory_unsafe"], - "embedding_model": "openrouter/..." -} -``` - -Retrieval packages must be bounded by role. A verifier gets the finding, relevant source, static corroboration, and counter-evidence candidates. It does not get hunter chain-of-thought. - -#### Vector Store Selection Criteria - -The vector store is a core ranking and false-positive-control dependency. The first implementation should choose one default after evaluating: - -```text -metadata filtering precision -incremental reindexing by repository snapshot -local persistence and portability -query latency on medium repositories -embedding model metadata support -hybrid lexical/vector retrieval support or integration path -deterministic export/import for scan reproducibility -operational simplicity for opencode users -``` - -Ranking depends on retrieval quality. Poor metadata filters will contaminate verifier context and increase false positives. The store must make it easy to retrieve "same symbol, same package, same vulnerability class, same evidence level" rather than semantically similar but irrelevant code. - -### Preprocessor - -The preprocessor produces `FileTarget` records. It should start simple and grow by plugins. - -Initial fields: - -```text -path -absolute_path -language -loc -tags -imports_by -transitive_callers -static_hints -sarif_hints -reachability_hints -has_fuzz_entry_point -defines_security_constants -``` - -Initial tags: - -```text -memory_unsafe -parser -crypto -deserialization -syscall_entry -network_entry -filesystem_entry -fuzzable -privileged -``` - -### Static Analysis Mapping Layer - -The static analysis layer turns tool output and skill discipline into a map for the harness. It is not only a source of finished findings. - -Primary mapping sources: - -```text -Semgrep: syntactic and lightweight semantic patterns, custom rule iteration, variant seeds -CodeQL: interprocedural dataflow, taint paths, structural vulnerability queries -Entry-point analysis: routes, CLI/process boundaries, parser inputs, smart-contract state-changing entry points, callbacks, privileged surfaces -Differential review: changed attack surfaces, regression-prone patches, new trust-boundary crossings -Sharp-edge review: unsafe APIs, insecure defaults, confusing configuration, misuse-prone interfaces -SARIF parsing: normalized evidence, rule metadata, locations, severity, fingerprints -``` - -Mapping outputs: - -```text -static_hints on FileTarget -sarif_hints on FileTarget -taint_paths and source/sink pairs -variant_search_seeds -changed_surface records -sharp_edge records -entry_point records -reachability_hints on FileTarget -evidence candidates for verifier -``` - -This layer should use Trail of Bits skills as task templates and checklists. For example, the CodeQL discipline supplies query/dataflow thinking, Semgrep supplies pattern and variant loops, entry-point analysis supplies attacker-controlled surface and access classification, differential review supplies changed-code blast-radius analysis, and sharp-edge review supplies API misuse mapping. - -### Benchmark Layer - -The benchmark layer is the quality control system for a realistic grounded finder. It prevents the scanner from passing narrow fixtures while missing obvious bugs in vulnerable projects. - -Benchmark manifest shape: - -```json -{ - "name": "nodegoat", - "language": "javascript", - "frameworks": ["node", "express"], - "source": {"type": "local", "path": "../benchmarks/nodegoat"}, - "setup": {"commands": [], "requires_network": false}, - "modes": ["quick", "standard"], - "expected": [ - { - "cwe": "CWE-89", - "class": "sql_injection", - "path": "app/routes/*.js", - "evidence": "user input reaches database query" - } - ], - "known_noise": [ - {"reason": "training-only route", "scope": "docs/**"} - ], - "baselines": ["semgrep", "codeql", "clearwing"] -} -``` - -Initial benchmark families: - -```text -javascript_node_web: NodeGoat, DVNA, OWASP Juice Shop slices, vulnerable Express fixtures -python_web: PyGoat, vulnerable Flask/Django apps, pickle/deserialization and command-injection fixtures -java_web: OWASP Benchmark, WebGoat, Juliet Java, Spring MVC injection/auth fixtures -c_cpp: Juliet C/C++, Damn Vulnerable C/C++ Program, vulnerable parsers, ASan/UBSan crash fixtures -``` - -Benchmark metrics: - -```text -expected_findings_total -matched_findings_total -missed_findings_total -unsupported_classes -false_positive_total -false_positive_reasons -recall_by_cwe -recall_by_language -precision_when_ground_truth_allows -runtime_seconds -model_calls_and_cost -external_baseline_delta -``` - -Benchmark misses are first-class calibration inputs. A miss should record the source location, expected CWE, which stage failed to surface it, whether an external baseline found it, and the next candidate improvement: static rule, SARIF source, entry-point mapping, CodeQL/Semgrep discipline, LLM hunter prompt, retrieval package, dynamic reproducer, or skill routing. - -#### Entry-Point Reachability Mapping - -Entry-point mapping is a Phase 5A prerequisite for useful ranking and false-positive reduction. Pattern matches and static analyzer alerts are not enough; the harness must identify whether the affected function or symbol is plausibly reachable from attacker-controlled input. - -Initial entry-point kinds: - -```text -route: HTTP/framework route handlers and request callbacks -cli: process entry points, argparse/click handlers, argv consumers, main functions -parser: parser or deserialization inputs that consume untrusted data -fuzz: explicit fuzz entry points such as LLVMFuzzerTestOneInput -smart_contract_state_change: externally callable non-view contract functions -contract_callback: callbacks and hook implementations invoked by other contracts -privileged_surface: auth boundaries, admin operations, filesystem/process/syscall boundaries -``` - -Initial access levels: - -```text -public -authenticated -role-restricted -contract-only/callback -local-only -review-required -``` - -Each entry-point record should include path, function name when known, start line, end line, name, kind, access level, trust boundary, access evidence, condition evidence, provenance, and rationale. Access level distinguishes unauthenticated/public from authenticated, role-restricted, local-only, callback-only, and review-required surfaces. Condition evidence captures feature flags, rollout toggles, pause guards, experiment gates, and runtime guard expressions. The preprocessor attaches matching records to `FileTarget.reachability_hints`. Findings use the source line to inherit only function-level entry points whose range contains the finding. The ranker uses these hints to raise file review priority for files containing public routes, parser/fuzz inputs, authenticated surfaces, contract callbacks, and role-restricted privileged surfaces. A finding outside a reachable function should remain eligible for review, but should not inherit the entry point's priority without call-graph evidence. - -### Ranker - -Ranking should combine heuristics, graph features, static hints, and model judgment. - -Scores: - -```text -surface: direct vulnerability likelihood, 1-5 -influence: downstream blast radius, 1-5 -reachability: attacker-controlled path likelihood, 1-5 -priority: surface * 0.5 + influence * 0.2 + reachability * 0.3 -``` - -The ranker should emit concise rationales: - -```json -{ - "path": "src/parser.c", - "surface": 5, - "influence": 4, - "reachability": 4, - "priority": 4.5, - "rationale": "C parser with external input, pointer arithmetic, and many callers", - "model_id": "openrouter/...", - "static_boosts": ["memory_unsafe", "parser", "sarif_high"] -} -``` - -Ranking is not only prioritization; it is a false-positive filter. The ranker should use verifier outcomes and dynamic evidence outcomes to calibrate which surfaces deserve attention. Repeated rejected patterns should be demoted unless new evidence appears. - -Ranking metrics: - -```text -verified findings per tier -false positives per tier -false-positive reason distribution -time-to-verification per tier -retrieval hit quality per role -missed known-vulnerable fixture targets -``` - -### False-Positive Elimination Loop - -HarnessX-style adjustment is the mechanism that reduces false positives over time. Each scan produces traces, verifier decisions, rejected hypotheses, static analyzer deltas, retrieval packages, and evidence transitions. The harness should use those outputs to adjust future runs. - -Adjustment targets: - -```text -ranker score calibration -retrieval filters and namespace weights -hunter prompt constraints -skill routing rules -verifier tie-breaker templates -static mapping seeds -variant search patterns -finding deduplication rules -``` - -False-positive records: - -```text -finding_id -rejection_reason -counter_evidence -scope of suppression or demotion -affected ranker/static/retrieval inputs -verifier model_id -human_override if present -``` - -False-positive learning must be scoped. A rejected claim in one framework, version, path, or call pattern should not globally suppress a vulnerability class. - -### Hunter Pool - -The hunter pool schedules targets by priority tier and depth. - -Suggested tiers: - -```text -A: priority >= 3.0, most budget -B: priority >= 2.0, moderate budget -C: priority < 2.0, opportunistic budget -``` - -Hunter inputs: - -```text -target metadata -bounded source context -static hints -retrieved related code -selected Trail of Bits skill snippets -sandbox capabilities -budget -evidence requirements -``` - -Hunter outputs must be structured and evidence-oriented. They should not rely on hidden reasoning. - -Hunters must be language- and framework-aware. The default hunter prompts should not ask one generic model to find every vulnerability type. Initial hunter families should cover: - -```text -javascript_node_web: injection, XSS, SSRF, prototype pollution, insecure deserialization, auth/session bugs, dependency misuse -python_web: command injection, SQL/template injection, deserialization, path traversal, Flask/Django misconfiguration, auth/session bugs -java_web: servlet/Spring injection, XXE, deserialization, path traversal, authorization bypass, crypto/API misuse -c_cpp: memory corruption, integer overflow, format strings, command execution, parser bugs, lifetime and ownership mistakes -``` - -### Verifier - -The verifier runs independently from the hunter. It receives only auditable facts: - -```text -finding claim -source excerpts -static evidence -retrieved relevant context -reproducer or crash artifacts when available -negative evidence candidates -``` - -Verifier output: - -```json -{ - "is_real": true, - "severity": "high", - "confidence": 0.82, - "evidence_level": "static_corroboration", - "pro_case": "...", - "counter_case": "...", - "tie_breaker": "Build a reproducer that drives len past buffer capacity", - "required_next_step": "attempt_crash_reproduction" -} -``` - -### Fusion Adjudication - -Fusion is the mandatory deepening mechanism whenever the task requires more reasoning depth than the normal ranker, hunter, verifier, and static mapping loop can provide. - -Fusion triggers: - -- Critical or high severity findings. -- Findings with verifier disagreement. -- Findings that gate patching or disclosure. -- Findings where static and semantic evidence conflict. -- Findings where a patch could introduce regression risk. -- Findings where multiple skills disagree on exploitability or remediation. -- User or task requests for deeper analysis, ultrathink-style review, or high-assurance adjudication. - -Fusion protocol: - -```text -panel A independently reviews bounded evidence -panel B independently reviews bounded evidence -panels critique each other -panels revise -panels vote/rank -decider issues final disposition -reconciler records accepted/rejected/blocked items -verifier records fresh evidence -``` - -Fusion should not be forced for routine low-severity findings when ordinary evidence is already sufficient. It must run when the task requires deeper analysis, when evidence does not converge, or when the decision gates a risky fix, exploit claim, disclosure, or high-impact report. - -### Docker Sandbox - -Sandbox defaults: - -```text -network: none -workspace: read-only -scratch: writable tmpfs or volume -cap_drop: all -cap_add: SYS_PTRACE only when sanitizer/debugger needs it -memory: bounded -cpu: bounded -pids: bounded -runtime: docker default, optional gVisor -``` - -Sandbox use cases: - -```text -static tool execution -build system probing -sanitizer builds -fuzz harness compilation -PoC execution -local service startup -netcat-style local socket probes -protocol smoke tests -patch validation in writable copy -``` - -Patch validation must use a writable copy, never the user's working tree by default. - -### Dynamic Analysis Boundary - -OpenUltraSAST is SAST-first, but dynamic analysis is in scope when it provides necessary evidence. The harness may run controlled runtime checks such as service startup, local HTTP/TCP probes, netcat-style reachability checks, PoC execution, and Clearwing-style network validation patterns. - -Dynamic analysis rules: - -```text -disabled by default -enabled only by explicit config or task-required evidence -scoped to declared hosts, ports, containers, and commands -logged as artifacts -bounded by timeout and resource limits -never treated as an open-ended network pentest loop -``` - -Dynamic evidence can support transitions such as `static_corroboration -> crash_reproduced`, `static_corroboration -> root_cause_explained`, or `root_cause_explained -> exploit_demonstrated` when the artifact actually demonstrates the claim. - -### Explain Mode And Interactive Fixing - -Explain mode turns a finding or benchmark miss into a prevention-oriented explanation. It should be useful to a coder who wants to understand the mistake, a vibecoder who needs safe guidance, and a reviewer who needs evidence boundaries. - -Explain input types: - -```text -finding_id within a scan -benchmark miss ID -vulnerability class plus source location -SARIF result ID imported into a scan -``` - -Explain output sections: - -```text -what was found -why this pattern is dangerous -what attacker-controlled path or reachability evidence exists -what evidence is missing -how to fix it safely -how to avoid recreating it -which validation command or artifact would prove the fix -which security skills were used for guidance -``` - -Explain mode must not inflate evidence. If a result is only a pattern match, it says so. If reachability is unknown, it names the missing call graph, route, CLI, parser, or dynamic artifact needed to prioritize the fix. - -Interactive fixing builds on explain mode and the patch oracle. It should ask for confirmation before patch proposal, select skills by language and vulnerability class, produce a minimal diff, run or list the required validation checks, and preserve all decisions as artifacts. It must not silently apply patches to the user's repository by default. - -### Trail Of Bits Skill Integration - -Trail of Bits skills should be indexed and selected by routing metadata. - -Examples: - -```text -C/C++ memory target -> c-review, address-sanitizer, libfuzzer, harness-writing -Python parser target -> atheris, property-based-testing -Rust target -> cargo-fuzz, zeroize-audit when secrets are present -Crypto target -> constant-time-analysis, wycheproof, vector-forge -SARIF processing -> sarif-parsing -Semgrep mapping loop -> semgrep, semgrep-rule-creator, semgrep-rule-variant-creator -CodeQL mapping loop -> codeql, sarif-parsing -Changed-code review -> differential-review, graph-evolution -API footgun review -> sharp-edges, insecure-defaults -Smart contract target -> chain-specific scanner skill -``` - -Do not paste entire skills into every prompt. Store descriptors, retrieval chunks, and short operating checklists. Pull only the relevant snippets into a role-specific context package. - -### UltraCode Fix Workflow - -OpenUltraCode discipline should govern fix authoring and fix audit. The patch oracle is not a single prompt that writes code and declares success. - -Fix workflow: - -```text -intake verified finding and evidence -create bounded fix plan -generate minimal patch in writable sandbox or worktree -run affected checks and reproducer -run adversarial fix review -reconcile accepted/rejected/deferred review items -run fresh verification -emit patch diff and validation report -``` - -Fix audit inputs: - -```text -original finding evidence -patch diff -differential analysis of changed code -Semgrep/CodeQL result delta when available -tests and reproducer output -sharp-edge regression checklist when API/config behavior changes -``` - -The workflow can propose a fix without applying it. It can only mark a fix as ready when the adversarial review is reconciled and fresh verification evidence exists. - -### Data Model - -Core entities: - -```text -Scan -RepositorySnapshot -HarnessRun -ProcessorTrace -ModelCall -FileTarget -Finding -Evidence -Artifact -PatchProposal -VerificationDecision -FusionDecision -FalsePositiveLearning -RankingCalibration -``` - -Finding fields: - -```text -id -scan_id -title -description -severity -confidence -status -evidence_level -path -start_line -end_line -symbol -vulnerability_class -attack_scenario -impact -remediation -discovered_by -verified_by -model_ids -artifact_refs -created_at -updated_at -``` - -Evidence levels: - -```text -suspicion -static_corroboration -crash_reproduced -root_cause_explained -exploit_demonstrated -patch_validated -``` - -### Artifact Layout - -Suggested scan directory: - -```text -.openultrasast/runs// - manifest.json - config.resolved.json - repo_snapshot.json - preprocess/ - index/ - rank/ - traces/ - model_calls/ - sandbox/ - findings.json - report.md - report.sarif - patches/ -``` - -## Implementation Strategy - -Build the system in vertical slices: - -1. Quick scan: intake, preprocess, rank, LLM/static finding generation, JSON/Markdown report. -2. Verification: independent verifier, evidence ladder enforcement, SARIF export. -3. Retrieval: OpenRouter embeddings, persistent index, bounded context packages. -4. Static mapping: Semgrep/CodeQL SARIF ingest, differential mapping, sharp-edge records. -5. False-positive loop: rejected finding taxonomy, ranking calibration, retrieval adjustment, and scoped suppressions. -6. Sandbox: Docker command runner, static tools, read-only source, scratch volume. -7. Dynamic evidence: controlled service startup, local probes, PoC execution, and artifact capture. -8. Deep mode: sanitizer builds, fuzz harness generation, crash artifacts. -9. Patch mode: UltraCode-style fix authoring, adversarial fix audit, validation loop. -10. MCP/opencode integration: narrow tools and commands. -11. Fusion: mandatory deepening for task-required hard findings, contradictory evidence, and risky fixes. - -## Key Design Decisions - -1. Evidence level is a state machine, not a label the model can freely assign. -2. Verifier context excludes hunter reasoning to reduce contamination. -3. OpenRouter model selection is role-scoped and auditable. -4. Embeddings and vector-store metadata filtering are part of the runtime, not an offline optional add-on. -5. Docker isolation is mandatory for execution-heavy modes. -6. Trail of Bits skills are retrieved as scoped expertise, not global prompt bloat. -7. MCP surface is narrow and project-oriented. -8. Fixes are authored and audited through an UltraCode-style workflow, not a one-shot patch prompt. -9. Fusion is the required deepening path when the task demands high-assurance reasoning, not a replacement for routine analysis. -10. Harness adjustment loops must reduce repeated false positives without hiding auditable evidence. - -## Risks - -1. OpenRouter embedding availability and model-specific response formats may vary. -2. Deep mode can become expensive without strict budget controls. -3. Docker sandbox escape risk requires conservative defaults and no-network execution. -4. LLM findings can inflate false positives unless the evidence ladder is enforced in code. -5. Multi-language callgraph quality will vary; ranking must tolerate partial graph data. -6. Skill retrieval can overfit prompts if descriptors are too broad. - -## Open Questions - -1. Which local vector store should be the default: Chroma, LanceDB, SQLite vector extension, or Qdrant local? -2. Should the first implementation be pure Python like Clearwing/HarnessX, or TypeScript to align with more opencode extension tooling? -3. Should OpenUltraSAST vendor a minimal HarnessX-inspired runtime or depend directly on HarnessX? -4. What minimum OpenRouter embedding model should be required for acceptable code retrieval? -5. Should patch proposals be applied only through git worktrees? diff --git a/.kiro/specs/openrouter-sast-harness/requirements.md b/.kiro/specs/openrouter-sast-harness/requirements.md deleted file mode 100644 index 2bb5b54..0000000 --- a/.kiro/specs/openrouter-sast-harness/requirements.md +++ /dev/null @@ -1,242 +0,0 @@ -# Requirements: OpenRouter SAST Harness - -> **Status 2026-10-02:** retired (see the note below). Its OpenCode skill `openultrasast-kiro-impl`, which still sent agents to this tasks.md, was deleted. Kept as decision history; nothing in src, tests or CI points here. - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Overview - -OpenUltraSAST shall be an independent OpenCode-driven security harness for finding, validating, reporting, and optionally fixing source-code vulnerabilities at scale. It shall use OpenRouter-selected chat models for reasoning, OpenRouter embeddings for semantic retrieval, Docker for isolated analysis, and an explicit evidence ladder to prevent AI suspicion from being reported as verified security impact. - -The project shall use Clearwing only as an implementation oracle. It shall not import Clearwing internals, fork Clearwing, or expose Clearwing's broad MCP surface. - -## Goals - -1. Provide a repeatable SAST harness that can scan local repositories and produce ranked, evidence-scored findings. -2. Make false-positive elimination a primary product goal by separating suspicion, corroboration, verification, reproduction, and patch validation. -3. Combine static analysis, semantic retrieval, LLM hunters, adversarial verification, and patch validation into one opencode-native workflow. -4. Keep model routing explicit through OpenRouter, with per-role model configuration and no silent substitution. -5. Preserve Docker isolation as a first-class security boundary for code execution, harness compilation, fuzzing, and patch validation. -6. Integrate Trail of Bits skills as scoped security expertise selected by language, framework, vulnerability class, mapping need, or verification need. -7. Emit machine-readable artifacts suitable for CI, triage, and later training or harness evolution. -8. Build detector quality against grounded vulnerable-project benchmarks for JavaScript/Node/web, Python/web, Java, and C/C++ instead of relying on toy fixtures or pattern-only tests. -9. Help coders and vibecoders understand and fix findings through an evidence-aware explain mode that teaches the secure pattern, not only flags the insecure one. - -## Completeness Model - -Implementation progress shall be reported against functional harness capability, not raw checklist percentage. A task may be complete as scaffold while the product capability remains incomplete. - -Completion levels: - -1. `scaffold`: schemas, CLI shape, local artifacts, tests, or isolated primitives exist. -2. `usable`: the feature runs end-to-end for a narrow local fixture without manual wiring. -3. `integrated`: the feature participates in the scan harness with traces, evidence, reports, and degradation handling. -4. `production`: the feature is robust enough for real repositories, CI budgets, failure recovery, and analyst audit. - -Progress reports shall identify both checklist progress and functional completeness. The project shall not describe OpenUltraSAST as a working harness until the `usable_harness_mvp` gate is satisfied. - -Functional gates: - -1. `scaffold_quick_scan`: quick scan can enumerate files, rank targets, emit basic static findings, and write JSON/Markdown artifacts. This gate does not constitute a security harness. -2. `usable_harness_mvp`: harness runtime, event tracing, static mapping ingestion, evidence ladder enforcement, independent verification, and JSON/Markdown/SARIF reporting work together on fixture repositories. -3. `standard_security_harness`: standard mode runs ranked hunters with retrieval packages, selected security skills, independent verification, false-positive learning, and auditable traces. -4. `sandboxed_dynamic_harness`: Docker sandbox, scoped dynamic probes, sanitizer/build artifacts, and evidence transitions work without hidden network or host mutation. -5. `fix_validation_harness`: patch proposals, adversarial fix audit, differential/static delta checks, and sandbox validation are integrated and cannot mark patches validated without passing evidence. -6. `opencode_product`: MCP tools, opencode commands/skills, fusion deepening, CI-ready reports, cost controls, and hardening are complete enough for routine opencode-driven use. -7. `benchmark_grounded_finder`: benchmark manifests, expected vulnerabilities, tool comparison baselines, recall/precision metrics, and missed-finding regression tracking exist for JavaScript, Python, Java, and C/C++ vulnerable projects. -8. `explain_and_fix_workflow`: users can ask for an evidence-grounded explanation of a finding, receive prevention guidance, and enter an interactive fixing workflow that routes to the appropriate security skills without silently applying patches. - -The current implementation state shall be documented as `usable_harness_mvp` until later gates are explicitly verified. - -## Non-Goals - -1. Do not make network pentesting the core workflow. -2. Do not reimplement all of Clearwing. -3. Do not report LLM-only claims as verified vulnerabilities. -4. Do not require a local embedding model. -5. Do not depend on a single OpenRouter model. -6. Do not mutate user repositories by default. - -## Functional Requirements - -### Repository Intake - -1. The system shall accept a local repository path as the primary scan target. -2. The system should later support git URLs, but initial implementation may require local checkout. -3. The system shall snapshot basic repository metadata: root path, commit hash when available, language mix, file count, and ignored paths. -4. The system shall respect common ignore files such as `.gitignore`, `.ignore`, and explicit harness config excludes. - -### Preprocessing - -1. The system shall enumerate source files and assign initial language, LOC, and path metadata. -2. The system shall tag files using lightweight static heuristics, including memory-unsafe code, parser surfaces, crypto surfaces, authentication boundaries, syscall or process boundaries, serialization boundaries, and fuzzable entry points. -3. The system shall build a dependency or call graph where practical. -4. The system shall ingest optional static analyzer output such as Semgrep SARIF or CodeQL SARIF. -5. The system shall preserve preprocessing artifacts for later ranking, retrieval, and audit. - -### Static Analysis And Mapping Discipline - -1. The system shall treat Semgrep, CodeQL, differential review, and sharp-edge/API review as mapping disciplines, not just optional external scanners. -2. The system shall ingest Semgrep and CodeQL output as SARIF and normalize it into static hints, evidence candidates, and variant-search seeds. -3. The system shall support function-level entry-point and reachability analysis to identify attacker-controlled inputs, externally callable functions, public routes, CLI/process boundaries, parser entry points, smart-contract state-changing entry points, callbacks, and privileged/admin-only surfaces. -4. The system shall classify entry points by access level and trust boundary, including public/unrestricted, authenticated, role-restricted, contract-only/callback, local-only, and unknown/review-required. -5. The system shall preserve condition evidence for entry points, including feature flags, rollout toggles, pause guards, experiment gates, and runtime guard expressions that affect whether a threat actor can reach the code. -6. The system shall use function-level reachability modeling to prioritize candidate bugs inside reachable entry-point functions and to demote pattern-only findings that lack a plausible attacker-controlled path. -6. The system shall support differential analysis between a base and head revision to identify changed attack surfaces, newly reachable code, and regression-prone fixes. -7. The system shall support sharp-edge mapping for APIs, configuration, defaults, cryptographic interfaces, and dangerous framework patterns that create predictable misuse. -8. The system shall preserve analyzer provenance so each finding can cite which static tool, rule, diff, entry-point analysis, or skill-derived checklist contributed evidence. - -### Benchmark-Grounded Detection - -1. The system shall maintain benchmark manifests for realistic vulnerable projects, not only synthetic single-file fixtures. -2. The initial benchmark matrix shall cover JavaScript/Node/web, Python/web, Java, and C/C++. -3. Benchmarks shall include vulnerable training projects and curated CWE samples such as NodeGoat, DVNA or similar Node/Express apps, OWASP Juice Shop slices, PyGoat or vulnerable Flask/Django samples, OWASP Benchmark/WebGoat/Juliet Java, Juliet C/C++, vulnerable parser fixtures, and small sanitizer-backed C/C++ crash targets. -4. Each benchmark manifest shall record repository path or source, language, framework, setup commands, scan modes to run, expected vulnerability classes, expected locations when known, and known noisy/non-security patterns. -5. The benchmark runner shall emit recall, precision where ground truth allows it, false-positive reason distribution, missed expected findings, unsupported vulnerability classes, runtime, model usage, and artifact paths. -6. Benchmark results shall compare quick, standard, and deep modes separately so rule coverage, LLM hunting, and dynamic validation are not conflated. -7. Benchmark runs shall support comparison against external baselines such as Clearwing, Semgrep, CodeQL, clang-tidy, npm audit, Bandit, or language-appropriate tools when their outputs are available as SARIF or normalized findings. -8. Missed benchmark vulnerabilities shall become regression records that feed rule expansion, ranking calibration, retrieval packages, hunter prompts, and skill routing. -9. A detector shall not be considered realistic for a language until it demonstrates useful recall on at least one realistic benchmark project and one curated CWE corpus for that language. -10. Benchmark artifacts shall be preserved under `.openultrasast/benchmarks//` with links to the scan artifacts that produced each metric. - -### OpenRouter Embeddings And Retrieval - -1. The system shall use OpenRouter embedding endpoints for semantic indexing. -2. The system shall chunk source code, docs, static findings, prior mechanisms, and Trail of Bits skill descriptors into separate collections or namespaces. -3. The system shall store vectors in a local persistent vector database by default. -4. The system shall support reusing an existing index when the repository snapshot has not changed. -5. The system shall retrieve bounded context packages for rankers, hunters, verifiers, and patchers. -6. The system shall evaluate vector-store candidates against ranking quality, metadata filtering, incremental indexing, local persistence, cost, and reproducibility before selecting a default. -7. The vector store shall support precise metadata filters for path, language, symbol, vulnerability class, evidence level, scan ID, and repository snapshot. - -### Ranking - -1. The system shall score candidate files on at least three axes: surface, influence, and reachability. -2. The system shall compute a composite priority using an explicit formula, initially `surface * 0.5 + influence * 0.2 + reachability * 0.3`. -3. The system shall keep low-surface high-influence files eligible for analysis. -4. The system shall combine static heuristics, dependency graph data, entry-point reachability, embeddings, and OpenRouter LLM scoring. -5. The system shall record score rationales and model identity. -6. The system shall measure ranking quality with fixture repositories and prior scan outcomes, including false-positive rate by tier and missed-true-positive rate where ground truth exists. -7. The system shall feed verifier outcomes back into ranking calibration so repeated false-positive patterns lose priority over time. - -### False-Positive Elimination Loop - -1. The system shall track every rejected, contradicted, duplicate, unsupported, and unverified finding as false-positive learning data. -2. The system shall record why a finding failed: unreachable path, missing attacker control, sanitizer disproved, static rule mismatch, incorrect model assumption, duplicate finding, or insufficient impact. -3. The system shall use HarnessX-style adjustment loops to update prompts, retrieval filters, ranking calibration, skill routing, verifier tie-breakers, and static mapping seeds from false-positive outcomes. -4. The system shall keep false-positive learnings auditable and reversible; learned suppressions shall cite evidence and scope. -5. The system shall prefer evidence-gated demotion over deletion so analysts can inspect why a suspected issue was suppressed or downgraded. - -### Harness Execution - -1. The system shall model the scan as a HarnessX-style harness composed of processors, tools, memory, sandbox, trace, and model roles. -2. The system shall expose stage hooks for task start, stage start, before model, after model, before tool, after tool, stage end, and task end. -3. The system shall contract-check processor inputs and outputs in strict mode for CI and warn mode for local exploration. -4. The system shall serialize harness configuration, processor versions, model roles, and prompt hashes into scan artifacts. - -### Hunter Pool - -1. The system shall fan out analysis across ranked targets using budgeted tiers. -2. The system shall support at least three analysis depths: quick, standard, and deep. -3. Quick mode shall rely on static analysis, ranking, and bounded LLM review without code execution. -4. Standard mode shall use sandboxed tool-calling hunters and adversarial verification. -5. Deep mode shall add harness generation, sanitizer builds, fuzzing, dynamic reproduction, crash reproduction, and patch validation where applicable. -6. The system shall record every hunter trajectory as structured JSONL. -7. Standard-mode hunters shall be evaluated against benchmark misses before being treated as production-quality detectors. -8. Hunter prompts shall be language- and framework-aware for JavaScript/Node/web, Python/web, Java, and C/C++ instead of using one generic vulnerability prompt for all targets. - -### Evidence Ladder - -1. Every finding shall carry exactly one current evidence level. -2. Evidence levels shall be ordered as: `suspicion`, `static_corroboration`, `crash_reproduced`, `root_cause_explained`, `exploit_demonstrated`, `patch_validated`. -3. Findings below `static_corroboration` shall not be reported as verified. -4. Exploit or PoC generation shall require at least `crash_reproduced` or equivalent trigger evidence. -5. Auto-patching shall require at least `root_cause_explained` and shall only advance to `patch_validated` after tests or reproducer validation pass. - -### Verification And Fusion - -1. The system shall verify findings using independent context that excludes hunter chain-of-thought and raw hunter reasoning. -2. The verifier shall steel-man both the vulnerability case and the false-positive case. -3. The verifier shall output explicit tie-breaker evidence requirements. -4. The system shall support OpenUltraCode-style fusion for task-required deepening, high-impact findings, uncertain findings, risky fixes, or unresolved disagreement, using two panels and an explicit decider. -5. Fusion output shall disclose model IDs, panel roles, degradations, warnings, votes, and final decision source. -6. Every adversarial finding shall receive a disposition: accepted, rejected, mitigated, deferred, or blocked. -7. Fusion shall be mandatory whenever the task requires deeper reasoning than the normal ranker, hunter, verifier, and static mapping loop can provide. -8. Fusion shall not be forced for routine low-risk issues unless the task explicitly asks for deeper analysis or the evidence remains unresolved. - -### Docker Sandbox - -1. The system shall run untrusted build, test, fuzz, and PoC commands inside Docker by default. -2. Analysis containers shall have network disabled by default. -3. Source mounts shall be read-only except in explicit patch workspaces. -4. Containers shall use a writable scratch volume or tmpfs. -5. Containers shall enforce CPU, memory, pids, timeout, and capability limits. -6. The system shall support sanitizer variants, initially ASan and UBSan for C/C++ targets. -7. The system should support gVisor or another hardened runtime when available. - -### Dynamic Analysis - -1. The system shall support optional dynamic analysis when runtime evidence is needed to verify, reproduce, or de-risk a finding. -2. Dynamic analysis may include service startup, local socket checks, netcat-style probes, protocol smoke tests, PoC execution, or Clearwing-style network validation patterns. -3. Dynamic network access shall be disabled by default and enabled only through explicit scan configuration or task-required evidence collection. -4. Dynamic probes shall run in sandboxed environments with declared network scope, timeouts, command logs, and captured artifacts. -5. Dynamic analysis shall contribute evidence but shall not turn OpenUltraSAST into a general network pentest agent. - -### Patch Oracle - -1. The system shall optionally generate minimal defensive patches for verified findings. -2. The patch workflow shall follow OpenUltraCode discipline: intake, bounded plan, implementation, adversarial review, reconciliation, and fresh verification. -3. The patch oracle shall target the smallest safe change. -4. The patch oracle shall not rewrite unrelated code or change public signatures unless required and approved. -5. Patch validation shall rerun the reproducer, affected tests, or relevant build checks in a writable sandbox. -6. Patch artifacts shall be emitted as diffs, not silently applied to the user's repository by default. -7. The system shall audit proposed fixes for regression risk using differential analysis and relevant Trail of Bits skills before marking them ready. - -### Explain Mode And Interactive Fixing - -1. The system shall expose an explain workflow for a finding, vulnerability class, or benchmark miss. -2. Explain mode shall describe the vulnerable pattern, why it matters, what attacker-controlled path or evidence exists, what evidence is missing, and what secure coding habit would have prevented it. -3. Explain mode shall distinguish education from verification; an explanation shall not upgrade evidence level unless the required evidence artifact already exists. -4. Explain mode shall cite source lines, finding artifacts, verifier results, SARIF/static evidence, dynamic evidence, benchmark metadata, and relevant skill snippets where available. -5. Explain mode shall support audience levels such as concise, learner, and reviewer so it can serve coders, vibecoders, and security reviewers without changing the underlying evidence. -6. Interactive fixing shall guide the user through a bounded plan, relevant security skill selection, minimal patch proposal, adversarial review, and validation commands. -7. Interactive fixing shall route by language and vulnerability class, for example C/C++ memory issues to `c-review`, `address-sanitizer`, `libfuzzer`, and `harness-writing`; JavaScript web issues to web/security mapping and dependency checks; Python web issues to injection/deserialization/auth/defaults checks; and Java issues to OWASP Benchmark/WebGoat-style dataflow and framework checks. -8. Interactive fixing shall never silently apply patches to the user's repository by default. It shall produce a proposed diff and required validation plan first. -9. Explain and fix outputs shall emphasize prevention patterns so users learn how not to recreate the same vulnerability class in future code. - -### Reporting - -1. The system shall emit JSON, SARIF, Markdown, and a manifest per scan. -2. Reports shall sort findings by severity, evidence level, reachability, and confidence. -3. Reports shall include model IDs, static evidence, verifier results, reproduction evidence, patch status, and artifact paths. -4. Reports shall distinguish suspected, verified, exploited, and patch-validated findings. - -### MCP And OpenCode Integration - -1. The project shall expose a narrow MCP server for OpenCode integration. -2. MCP tools shall focus on project-level operations: scan, status, findings, evidence, artifacts, patch proposal, and report export. -3. The MCP server shall not expose arbitrary internal tools as public MCP methods. -4. OpenCode commands shall invoke the harness with explicit model role configuration and visible assumptions. -5. MCP and OpenCode integration shall include narrow operations for benchmark status, finding explanation, and interactive patch proposal without exposing arbitrary shell execution. - -## Non-Functional Requirements - -1. The system shall be deterministic where possible: fixed config, artifact manifests, prompt hashes, and model identifiers. -2. The system shall degrade visibly when optional tools, Docker, embeddings, or specific models are unavailable. -3. The system shall avoid hidden network access from analysis containers; any dynamic network access shall be explicit, scoped, logged, and artifact-backed. -4. The system shall support CI usage with bounded runtime and cost budgets. -5. The system shall keep findings auditable from report back to source, trace, model call, verifier decision, and artifact. - -## Acceptance Criteria - -1. `scaffold_quick_scan`: A user can run a local repository scan in quick mode and receive ranked static findings with JSON and Markdown reports. This may use heuristics and static patterns only. -2. `usable_harness_mvp`: A user can run quick or standard fixture scans through the harness runtime and inspect event traces, normalized static mapping evidence, verifier decisions, and report artifacts. -3. `standard_security_harness`: A user can run standard mode and receive adversarially verified findings with explicit evidence levels, selected retrieval context, skill routing, and false-positive learning records. -4. `sandboxed_dynamic_harness`: A user can run deep mode on a small C/C++ parser target and receive sandboxed build/fuzz artifacts when the target is fuzzable. -5. `fix_validation_harness`: A patch cannot be marked validated unless a sandboxed validation command passes and the fix audit has no accepted blocking findings. -6. `opencode_product`: A user can drive scans, triage, evidence inspection, report export, fusion, and patch proposal through narrow MCP/OpenCode interfaces without exposing arbitrary shell execution. -7. A finding cannot be marked verified unless it reaches at least `static_corroboration` through enforced evidence transitions. -8. A scan manifest records repository snapshot, harness config, OpenRouter model IDs, embedding model, prompt hashes, processor versions, evidence transitions, and artifact paths. -9. `benchmark_grounded_finder`: A user can run a benchmark manifest for JavaScript, Python, Java, or C/C++ and receive language-level recall, missed finding, false-positive, runtime, and artifact reports. -10. `explain_and_fix_workflow`: A user can request an explanation for a finding or benchmark miss and receive evidence-cited prevention guidance plus an optional interactive fix plan without automatic repository mutation. diff --git a/.kiro/specs/openrouter-sast-harness/spec.json b/.kiro/specs/openrouter-sast-harness/spec.json deleted file mode 100644 index cdd6815..0000000 --- a/.kiro/specs/openrouter-sast-harness/spec.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "feature_name": "openrouter-sast-harness", - "language": "en", - "status": "retired", - "updated_at": "2026-10-02T00:00:00Z", - "retired_note": "2026-09-30: HarnessX was removed by harnessx-removal; this spec had no spec.json before 2026-10-02 and is kept as decision history." -} diff --git a/.kiro/specs/openrouter-sast-harness/tasks.md b/.kiro/specs/openrouter-sast-harness/tasks.md deleted file mode 100644 index cb5c53c..0000000 --- a/.kiro/specs/openrouter-sast-harness/tasks.md +++ /dev/null @@ -1,303 +0,0 @@ -# Tasks: OpenRouter SAST Harness - -## Completion Gates - -Checklist completion is not product completion. Each phase now carries a functional gate label: - -- `scaffold`: isolated primitives, schemas, local artifacts, or tests exist. -- `usable`: the feature works end-to-end for fixtures without manual wiring. -- `integrated`: the feature participates in the harness runtime, traces, evidence, and reports. -- `production`: the feature is robust for real repositories, CI budgets, and analyst audit. - -Current verified gate: `usable_harness_mvp`. - -Do not report OpenUltraSAST as a working security harness until `usable_harness_mvp` is checked. - -## Phase 0: Project Skeleton (`scaffold`, complete) - -- [x] Create Python package skeleton with CLI entry point `ousast`. -- [x] Add config loading for `openultrasast.toml` and environment variables. -- [x] Add resolved config artifact writer. -- [x] Add scan run directory creation under `.openultrasast/runs//`. -- [x] Add basic tests for config loading and run directory creation. - -## Phase 1: Intake And Preprocess - -- [x] Implement repository snapshot collection. -- [x] Implement source file enumeration with ignore handling. -- [x] Implement language detection from extension and shebang. -- [x] Implement LOC counting. -- [x] Implement initial tag heuristics for parser, memory unsafe, crypto, auth boundary, deserialization, syscall, network, filesystem, and fuzzable targets. -- [x] Emit `preprocess/file_targets.json`. -- [x] Add tests for tag heuristics on small fixture repositories. - -## Phase 2: Ranking - -- [x] Define `FileTarget` and ranking schemas. -- [x] Implement heuristic score floors and boosts. -- [x] Implement OpenRouter chat client for ranker role. -- [x] Implement chunked ranker calls with JSON response validation. -- [x] Compute composite priority. -- [x] Emit ranking rationales and model metadata. -- [x] Add tests for composite priority and malformed model response handling. - -## Phase 3: Quick Scan Reports - -- [x] Implement static finding schema. -- [x] Implement quick-mode hunter prompt and bounded file context. -- [x] Implement structured finding extraction. -- [x] Enforce initial evidence level `suspicion` unless static corroboration exists. -- [x] Emit `findings.json`. -- [x] Emit Markdown report. -- [x] Add smoke test against a fixture repository with one known insecure pattern. - -## Phase 4: OpenRouter Embeddings - -- [x] Implement OpenRouter embedding client. -- [x] Run a vector-store bakeoff for metadata filtering, local persistence, incremental indexing, export/import, and retrieval quality. -- [x] Select default local vector store based on bakeoff results. -- [x] Implement code chunking with path, symbol, and line metadata. -- [x] Implement namespaces for repo code, docs, static findings, mechanisms, skills, and traces. -- [x] Implement index reuse keyed by repository snapshot and embedding model. -- [x] Implement bounded retrieval package builder per role. -- [x] Add tests for chunk metadata and retrieval filters. - -## Phase 4A: Ranking Quality And False-Positive Loop - -- [x] Define false-positive reason taxonomy. -- [x] Add false-positive learning records for rejected, duplicate, unsupported, contradicted, and unverified findings. -- [x] Track ranking metrics by tier: verified findings, false positives, time to verification, and missed fixture vulnerabilities. -- [x] Feed verifier outcomes into ranking calibration. -- [x] Feed rejected findings into retrieval filter adjustment and prompt constraints. -- [x] Add tests proving scoped false-positive demotion does not globally suppress the vulnerability class. - -## Gate: `scaffold_quick_scan` - -- [x] Quick scan can enumerate files, rank targets, emit basic static findings, and write JSON/Markdown artifacts. -- [x] Quick scan is uv-managed and covered by unit tests. -- [ ] Quick scan includes real LLM hunter execution. This is intentionally deferred to standard mode. -- [x] Quick scan includes SARIF output. -- [x] Quick scan includes enforced verifier decisions. - -## Gate: `usable_harness_mvp` - -- [x] Harness runtime drives scan stages instead of direct CLI orchestration. -- [x] Event traces are emitted for every scan stage. -- [x] Static mapping ingestion feeds FileTarget hints and verifier evidence candidates. -- [x] Entry-point reachability feeds FileTarget reachability hints and ranking priority. -- [x] Evidence ladder state machine prevents invalid verified statuses. -- [x] Independent verifier runs on fixture findings without hunter reasoning. -- [x] JSON, Markdown, SARIF, and manifest artifacts share finding IDs and evidence references. -- [x] A fixture scan demonstrates one accepted static-corroborated finding and one rejected false positive. - -## Phase 5: Harness Runtime (`usable_harness_mvp` prerequisite) - -- [x] Define harness event model. -- [x] Define processor interface and contract checks. -- [x] Implement strict and warn modes. -- [x] Add trace writer for events and processor state changes. -- [x] Serialize harness config, prompt hashes, processor versions, and model roles. -- [x] Add tests for processor contract violations. - -## Phase 5A: Static Mapping Disciplines - -- [x] Add SARIF ingestion for Semgrep and CodeQL outputs. -- [x] Normalize SARIF rules, locations, severity, fingerprints, and provenance into static hints. -- [x] Add Semgrep mapping task records for pattern, variant, and rule-authoring loops. -- [x] Add CodeQL mapping task records for source, sink, sanitizer, and path explanations. -- [x] Add entry-point mapping records for routes, CLI/process boundaries, parser entry points, smart-contract state-changing entry points, callbacks, and privileged surfaces. -- [x] Classify entry points by access level: public, authenticated, role-restricted, contract-only/callback, local-only, or review-required. -- [x] Map entry points to function-level source ranges and access evidence. -- [x] Preserve conditional reachability evidence for feature flags, rollout toggles, pause guards, and runtime guard expressions. -- [x] Attach reachability hints to FileTargets and ranking inputs. -- [x] Annotate findings with function-level reachability status before prioritizing fixes. -- [x] Add differential mapping records for changed files, changed functions, and changed trust-boundary crossings. -- [x] Add sharp-edge records for insecure defaults, dangerous APIs, confusing config, and misuse-prone interfaces. -- [x] Add tests for mapping analyzer output to `FileTarget` hints and verifier evidence candidates. - -## Phase 6: Verification (`usable_harness_mvp` prerequisite) - -- [x] Define evidence ladder state machine. -- [x] Implement independent verifier prompt and schema. -- [x] Ensure verifier context excludes hunter reasoning. -- [x] Implement pro-case, counter-case, tie-breaker, and required-next-step fields. -- [x] Enforce that verified report status requires at least `static_corroboration`. -- [x] Add tests for invalid evidence transitions. - -## Gate: `standard_security_harness` - -- [x] Standard mode uses ranked hunter scheduling, retrieval packages, selected skill snippets, and independent verification. -- [x] Every hunter trajectory is persisted and linked to finding artifacts. -- [x] False-positive learning updates ranking calibration from verifier outcomes. -- [ ] Fusion can be triggered for high-impact, contradictory, or difficult findings. -- [ ] Standard mode uses real language-aware LLM hunters instead of reusing quick static findings. - -## Gate: `benchmark_grounded_finder` - -- [ ] Benchmark manifests exist for JavaScript/Node/web, Python/web, Java, and C/C++ vulnerable projects. -- [ ] Benchmark runner executes quick, standard, and deep modes as supported and preserves benchmark artifacts. -- [ ] Benchmark reports include recall, missed expected findings, false-positive reasons, runtime, and external baseline deltas. -- [ ] Missed benchmark findings feed calibration records for rules, ranking, retrieval, hunters, SARIF ingestion, dynamic probes, or skill routing. -- [ ] At least one realistic vulnerable project and one curated CWE corpus are tracked per supported language family. - -## Phase 6A: Benchmark Corpus And Scoreboard (`benchmark_grounded_finder` prerequisite) - -- [x] Define benchmark manifest schema with source, language, framework, setup, expected vulnerabilities, known noise, scan modes, and optional baselines. -- [x] Implement benchmark run directory creation under `.openultrasast/benchmarks//`. -- [x] Implement finding-to-ground-truth matching by CWE, rule ID, file, function, sink, and evidence text. -- [x] Implement benchmark metrics for recall, precision when possible, false-positive reasons, missed findings, runtime, model usage, and artifact links. -- [x] Add benchmark fixtures or manifests for JavaScript/Node/web, Python/web, Java, and C/C++. -- [x] Add external baseline ingestion for SARIF or normalized findings from Clearwing, Semgrep, CodeQL, clang-tidy, Bandit, npm audit, or language-appropriate tools. -- [x] Add tests proving benchmark misses are recorded as calibration inputs without automatically suppressing unrelated findings. - -## Gate: `explain_and_fix_workflow` - -- [ ] Explain mode can explain a finding or benchmark miss with source evidence, reachability status, missing evidence, secure pattern, and prevention guidance. -- [ ] Explain mode supports concise, learner, and reviewer audience levels without changing evidence levels. -- [ ] Interactive fixing selects relevant security skills and proposes a minimal patch plan without silently mutating the target repository. -- [ ] Fix guidance links to validation commands and required evidence transitions. - -## Phase 6B: Explain Mode And Interactive Fixing (`explain_and_fix_workflow` prerequisite) - -- [ ] Add explain artifact schema for finding ID, source refs, evidence refs, audience level, prevention guidance, selected skills, and validation advice. -- [ ] Add `ousast explain --scan ` CLI command. -- [ ] Add explain support for benchmark misses and imported SARIF findings. -- [ ] Implement language/vulnerability skill routing for JavaScript/Node/web, Python/web, Java, and C/C++. -- [ ] Add interactive fixing plan schema with confirmation gates, selected skills, proposed patch scope, validation commands, and evidence requirements. -- [ ] Ensure interactive fixing cannot mark patches validated unless the patch oracle and sandbox validation gates pass. -- [ ] Add tests that explanation does not upgrade evidence level and patch guidance does not mutate repositories by default. - -## Phase 7: Docker Sandbox (`sandboxed_dynamic_harness` prerequisite) - -- [ ] Implement Docker availability probe. -- [ ] Implement no-network read-only workspace sandbox runner. -- [ ] Add explicit scoped-network sandbox mode for task-required dynamic probes. -- [ ] Add scratch volume or tmpfs support. -- [ ] Add CPU, memory, pids, timeout, and capability limits. -- [ ] Add command execution artifact capture. -- [ ] Add sanitizer build environment for C/C++ fixtures. -- [ ] Add tests that verify workspace remains read-only. - -## Phase 7A: Dynamic Evidence (`sandboxed_dynamic_harness` prerequisite) - -- [ ] Implement dynamic analysis configuration with `enabled = false` by default. -- [ ] Implement service startup command records with declared ports and timeouts. -- [ ] Implement netcat-style local socket probe tool inside sandbox scope. -- [ ] Implement HTTP or protocol smoke-test artifact capture. -- [ ] Tie dynamic artifacts to evidence transitions without allowing model-only upgrades. -- [ ] Add tests that reject undeclared network targets. - -## Phase 8: Standard Mode Hunter Pool (`standard_security_harness` prerequisite) - -- [x] Implement tier assignment by priority. -- [x] Implement budget allocation and per-target limits. -- [x] Implement parallel hunter scheduling. -- [x] Persist hunter trajectories as JSONL. -- [x] Integrate retrieval packages and selected skill snippets. -- [x] Integrate verifier after hunter output. -- [x] Add fixture scan demonstrating at least one verified static-corroborated finding. - -## Phase 9: SARIF And CI Output (`usable_harness_mvp` prerequisite) - -- [x] Implement SARIF report writer. -- [x] Implement machine-readable manifest. -- [x] Add severity, confidence, evidence level, model IDs, and artifact refs to reports. -- [x] Add CI-friendly exit code policy. -- [x] Add tests for SARIF schema validity. - -## Gate: `sandboxed_dynamic_harness` - -- [ ] Docker sandbox runs build/test/fuzz/PoC commands with no network by default. -- [ ] Scoped dynamic probes require explicit declared targets and emit artifacts. -- [ ] Sanitizer/fuzz artifacts can advance evidence only through valid transitions. - -## Phase 10: Deep Mode (`sandboxed_dynamic_harness` prerequisite) - -- [ ] Implement fuzzable target detection beyond existing fuzz entry points. -- [ ] Implement fuzz harness generation prompt and schema. -- [ ] Compile generated harnesses in sandbox. -- [ ] Run libFuzzer or equivalent with sanitizer instrumentation. -- [ ] Capture crashes and minimization artifacts. -- [ ] Advance findings to `crash_reproduced` only with artifact evidence. -- [ ] Add a small C/C++ parser fixture with reproducible sanitizer crash. - -## Gate: `fix_validation_harness` - -- [ ] Patch proposals are generated as diffs and not silently applied. -- [ ] Fix lifecycle records intake, plan, implementation, adversarial review, reconciliation, verification, and ready states. -- [ ] Patch validation requires sandboxed checks and accepted adversarial findings must be resolved. - -## Phase 11: Patch Oracle (`fix_validation_harness` prerequisite) - -- [ ] Implement patch proposal schema. -- [ ] Implement OpenUltraCode-style fix lifecycle states: intake, plan, implement, adversarial_review, reconcile, verify, ready. -- [ ] Run patching in a writable sandbox copy or git worktree. -- [ ] Generate minimal diffs only. -- [ ] Rerun reproducer and affected checks. -- [ ] Run differential fix audit against the generated patch. -- [ ] Run Semgrep or CodeQL delta checks when corresponding evidence exists. -- [ ] Run sharp-edge fix audit when API, config, auth, crypto, or defaults are changed. -- [ ] Advance findings to `patch_validated` only after validation passes. -- [ ] Add tests that reject patches changing unrelated files. - -## Gate: `opencode_product` - -- [ ] Narrow MCP tools cover scan, status, findings, evidence, artifacts, patch proposal, and report export. -- [ ] OpenCode workflows can drive scan, triage, fusion, and patch audit without arbitrary shell exposure. -- [ ] Cost controls, retries, redaction, docs, examples, and CI gates are in place. -- [ ] OpenCode workflows can drive benchmark runs, finding explanations, and interactive fix proposals. - -## Phase 12: MCP And OpenCode Commands (`opencode_product` prerequisite) - -Implemented in `mcp.py` (zero-dep stdio JSON-RPC server) + the `ousast mcp` CLI -subcommand; tests in `tests/test_mcp.py`; README "MCP server" section documents it. - -- [x] Add project-local opencode skills for scan, triage, fix audit, and implementation steering. -- [x] Implement narrow MCP server. -- [x] Add `openultrasast.scan`, `status`, `findings`, `get_finding`, `evidence`, `artifacts`, `benchmark`, `explain`, `propose_patch`, and `export_report` tools. -- [x] Add opencode command docs for quick, standard, deep, verify, and patch workflows. (README; deep/patch noted as roadmap.) -- [x] Ensure MCP tools never expose arbitrary shell execution. (Surface-safety test asserts no shell/exec/command tool.) -- [x] Add integration test for MCP tool listing and scan status. - -## Phase 13: Fusion (`standard_security_harness` and `opencode_product` prerequisite) - -Implemented in `fusion.py` (zero-dep deterministic engine + lazy LLM panels), wired -into the standard-mode CLI (`fusion.json` + manifest `fusion` block), config `[fusion]`. -Tests in `tests/test_fusion.py`. - -- [x] Implement fusion trigger policy. -- [x] Trigger fusion whenever a task requires deeper reasoning, including difficult findings, contradictory evidence, high-impact decisions, and risky fixes. -- [x] Implement panel A, panel B, and decider role config. -- [x] Implement cross-critique and revised answer flow. -- [x] Implement vote/rank output schema. -- [x] Implement reconciliation dispositions. -- [x] Record model IDs, degradations, warnings, and final decision source. -- [x] Add tests for fusion output validation. - -## Phase 14: Skill Index (`standard_security_harness` prerequisite) - -Implemented in `skills.py` (descriptor inventory + scoring router + retrieval-namespace -chunking + budget-capped context); the HarnessX hunter prompt consumes the routed -skill context; tests in `tests/test_skills.py`. - -- [x] Build Trail of Bits skill descriptor inventory. -- [x] Chunk selected skill operating guidance into retrieval namespace. -- [x] Implement skill router by language, tags, vulnerability class, and stage. -- [x] Add prompt budget caps for skill snippets. -- [x] Add tests for routing C parser targets to memory/fuzzing skills and crypto targets to constant-time/vector skills. - -## Phase 15: Hardening And Release Readiness (`production` prerequisite) - -Implemented: `redaction.py` (traces/reports), provider retry/backoff in -`provider/openrouter.py`, `[hardening]` config (redact_secrets + max_findings budget; -agentic spend bounded by `[harnessx]`), `docs/threat-model.md`, `docs/examples.md`. -Tests in `tests/test_redaction.py`, `tests/test_provider_retry.py`, -`tests/test_hardening_pipeline.py`. CI gates pre-existed (ruff/mypy/pytest/detection -gate/dagger). - -- [x] Add cost budget enforcement. -- [x] Add model call retry and backoff policies. -- [x] Add redaction for secrets in traces and reports. -- [x] Add docs for threat model and sandbox limits. -- [x] Add end-to-end examples. -- [x] Add lint, typecheck, test, and CI gates. diff --git a/.kiro/specs/real-world-vfc-slice/brief.md b/.kiro/specs/real-world-vfc-slice/brief.md deleted file mode 100644 index 5b6e61c..0000000 --- a/.kiro/specs/real-world-vfc-slice/brief.md +++ /dev/null @@ -1,58 +0,0 @@ -# Brief: real-world-vfc-slice - -## Problem - -Operators care whether the harness fires on a real buggy snapshot and stays quiet after the patch (Firefox, Chromium, OpenSSL), not whether OWASP BenchmarkTest00074 is a TP. The labeled SAST slice is 11 textbook files. Overlay Youden there is **worse** than inventory (+9.09% → 0%). We cannot tell if the tool got better on messy product code. - -## Current State - -Slices today: `local` (CI 100%), `github` (6 vendored functions: SVEN, hutool, node-serialize), `sast` (OWASP/Juliet). `datasets.toml` already points at CVEfixes, MoreFixes, CWE-Bench-Java, Vul4J. Nothing is harvested from gecko-dev / Firefox. Whole mozilla-central is not in-tree (and should not be). FixFox (Zenodo) is embargoed until 2030. Firefox security commit messages are often scrubbed of "security" words, so grep-the-log is a bad miner. - -Measured on the current 11-pair **sast** slice (2026-09-04): - -| Scorer | pair_correct | vuln recall | silent-on-fix | Youden | -|--------|--------------|-------------|---------------|--------| -| Inventory (regex) | 2/11 (18%) | 54.5% | 54.5% | **+9.09%** | -| Overlay (current) | 2/11 (18%) | 45.5% | 54.5% | **0%** | - -The only overlay change vs inventory on that slice: `owasp-python-codeinj` went from LEAK (regex hit both sides) to MISS (ConfigParser heap → `flow_incomplete`, no promote). C/Java Juliet goodG2B still leak because those languages are unadjudicated and scoring falls back to inventory. - -## Desired Outcome - -A new pair slice (e.g. `vfc`) of **reviewed, isolated function pairs** from large projects, including Firefox/gecko-dev when a public fix commit exists in CVEfixes/MoreFixes. Offline CI. Honesty dashboard like `github`, not a 90% smoke gate. Overlay scoring once tree-sitter can parse C/JS/Java; inventory scoring until then. Provenance on every file (repo, commit, CVE, license). - -## Approach - -Do not clone Firefox. Filter public VFC databases for `gecko-dev` / `mozilla-firefox/firefox`, `chromium`, `openssl`, `curl`, plus existing CWE-Bench-Java. Extract the changed function before/after. Human-review labels (DiverseVul ~60% is not ground truth). Cap the runnable catalog (tens of pairs, not thousands). Prefer injection/XSS/path CWEs the overlay can talk about; keep UAF/buffer overflows as known inventory-only leaks until a later memory-safety spec. - -## Scope - -- In: harvest script or documented procedure; catalog slice; provenance headers; overlay-vs-inventory scoreboard on that slice; dataset.toml pointers for Firefox/Chromium. -- Out: Vendoring mozilla-central; live Bugzilla scraping; making Youden a merge gate; buffer-size semantics; tree-sitter packaging (owned by `tree-sitter-overlay-extra`). - -## Boundary Candidates - -- Catalog schema / slice name vs pair eval scoring -- Harvest tooling vs vendored excerpts - -## Out of Boundary - -- Parser extra, Joern, improve-loop auto-facts, `LANGUAGE_MANIFESTS` - -## Upstream / Downstream - -- Upstream: `pairs.py` catalog format; `propose-adjudicate-prove` overlay scoring -- Downstream: improve-loop signals from real misses; later memory-safety spec can reuse the same slice - -## Existing Spec Touchpoints - -- Extends: pair catalog only (new slice). Does not change smoke gate. -- Adjacent: `tree-sitter-overlay-extra` (C/Java overlay on this slice once the extra exists) - -## Constraints - -Isolated functions, same `relpath` vuln/fixed. License in the header. No 16 GB dumps in git. Firefox pairs only when a public git commit pair exists; MFSA text without a commit is not a pair. - -## Closure (2026-09-04) - -Closed. All tasks done; `tests/test_pair_corpus.py` and `tests/test_vfc_harvest.py` pass (17). Seeds from OpenSSL, Firefox, Chromium, and curl landed (176 pairs). Residuals found in review are corpus quality and scorer fidelity, not seed coverage: 25 curl pairs start mid-comment because harvest anchors on a comment mention; 129 expected rows have `sink = "unknown"`; the overlay column equals inventory because C facts have no memory sink. These are owned by `pair-corpus-honesty`, which supersedes this spec on the corpus and scorer plane. Do not reopen. diff --git a/.kiro/specs/real-world-vfc-slice/design.md b/.kiro/specs/real-world-vfc-slice/design.md deleted file mode 100644 index 3b8f315..0000000 --- a/.kiro/specs/real-world-vfc-slice/design.md +++ /dev/null @@ -1,376 +0,0 @@ -# Design Document - -## Overview - -**Purpose.** This feature gives OpenUltraSAST a labeled vuln-versus-fix slice from real OpenSSL, Firefox, and Chromium functions so engineers can evolve the harness against product code instead of only OWASP/Juliet. - -**Users.** Maintainers grow and score the catalog. AppSec engineers read the honesty dashboard. CI proves the catalog loads and the slice command runs offline. - -**Impact.** A new `vfc` slice is concatenated into the pair catalog. Overlay and inventory scores are reported side by side for that slice. The local 100% pair gate and the stage-1 smoke gate do not change. - -### Goals - -- Seed at least one reviewed isolated function pair each from OpenSSL, Firefox, and Chromium, with provenance. -- Score `vfc` as an honesty dashboard with overlay versus inventory figures. -- Harvest one function from public parent/fix blobs without cloning megarepos. -- Keep CI extra-free and offline; vfc Youden is not a merge condition. - -### Non-Goals - -- Cloning mozilla-central, gecko-dev, or Chromium. -- FixFox, Bugzilla scraping, or vendoring VFC dumps. -- CWE-121 buffer-size semantics, inter-file taint, or parser extra packaging. -- Changing inventory rules so the seed pairs become pair_correct. -- Making overlay or vfc Youden fail CI. - -## Boundary Commitments - -### This Spec Owns - -- The `vfc` pair slice: catalog rows, vendored excerpts, provenance headers, recipes. -- Harvest procedure that extracts one function from public parent/fix snapshots. -- Dual overlay/inventory scoreboard fields for the vfc slice. -- CLI slice choice `vfc` and tests that the slice loads and runs. -- Dataset pointers for OpenSSL, Firefox/gecko-dev, and Chromium. - -### Out of Boundary - -- Overlay dispositions, taint, facts, FileIR (`propose-adjudicate-prove`). -- Tree-sitter extra packaging (`tree-sitter-overlay-extra`). -- Stage-1 `LANGUAGE_MANIFESTS` detection gate and local 100% pair gate fail condition. -- Pattern rule bodies; do not retune memcpy/eval to chase vfc Youden. -- Memory-safety / CWE-121 semantics (later spec may reuse these pairs). - -### Allowed Dependencies - -- Existing `PairCase` / `load_pair_catalog` / `evaluate_pair` / `result_payload`. -- Overlay pair path already used by `sast` (inventory fallback when unadjudicated). -- Stdlib HTTP client for harvest (maintainer-only, not CI). -- Public hosts: GitHub raw blobs, hg.mozilla.org / Phabricator raw diffs. - -### Revalidation Triggers - -- Changing `PairCase` required fields or catalog concatenation order. -- Failing CI on any slice other than `local`. -- Adding vfc names to `LANGUAGE_MANIFESTS`. -- Making harvest a network-using CI step. -- Importing parser wheels into harvest or pair eval. - -## Architecture - -### Existing Architecture Analysis - -Pair eval materializes each side of a pair into an isolated temp tree and scans it. `local` and `github` use inventory (`quick_scan`). `sast` uses overlay, falling back to inventory when no promote/demote/coverage occurred. Catalog load already concatenates a second file. The pair gate prints every slice but fails only when `local` pair_pass_rate is below 100%. - -### Architecture Pattern and Boundary Map - -Selected: **concatenated honesty slice + dual scoreboard**. Same catalog schema. Overlay eval generalized from `sast` to `{sast, vfc}`. Inventory sidecar computed only for vfc. - -```mermaid -flowchart TD - cli[ousast pairs slice vfc] - load[load_pair_catalog] - mainCat[pairs catalog] - sastCat[sast catalog] - vfcCat[vfc catalog] - eval[evaluate_pair] - overlay[overlay with inventory fallback] - inventory[inventory only] - board[result payload] - gate[pair_gate local only] - cli --> load - load --> mainCat - load --> sastCat - load --> vfcCat - load --> eval - eval --> overlay - eval --> inventory - overlay --> board - inventory --> board - load --> gate -``` - -**Key decisions** - -- Primary `per_slice.vfc` metrics come from overlay-with-fallback (same as sast). -- `scorers.vfc.overlay` and `scorers.vfc.inventory` are explicit so operators can compare. -- Harvest never runs in pytest. Seed excerpts are committed. -- Dependency direction: recipes/excerpts → vfc catalog → pair loader → eval → CLI payload. Harvest writes excerpts; it does not import overlay. - -### Technology Stack - -| Layer | Choice / Version | Role in Feature | Notes | -|-------|------------------|-----------------|-------| -| CLI | existing `ousast pairs --slice` | Operator entry | Add `vfc` to choices | -| Catalog | TOML `[[pair]]` | Same schema as github/sast | New file, concatenated | -| Scoring | existing overlay + inventory | Dual scoreboard | No new taint engine | -| Harvest | Python stdlib urllib | Maintainer fetch | Not a core package dep | -| Tests | pytest extra-free | Offline existence + CLI | No network | - -## File Structure Plan - -``` -benchmarks/pairs/ -├── catalog.toml # unchanged rows; still the default path -├── datasets.toml # add OpenSSL, Firefox, Chromium pointers -├── README.md # document vfc slice -├── sast/catalog.toml # unchanged -└── vfc/ - ├── catalog.toml # slice = vfc rows - ├── recipes.toml # harvest inputs for seed and later pairs - ├── harvest.py # extract one function from parent/fix URLs - ├── openssl-cve-2014-0160/ - │ ├── vuln.c - │ └── fixed.c - ├── firefox-cve-2020-15667/ - │ ├── vuln.c - │ └── fixed.c - └── chromium-cve-2019-5786/ - ├── vuln.cc - └── fixed.cc -``` - -### Modified Files - -- `src/openultrasast/pairs.py`: concatenate vfc catalog; overlay path for `vfc`; `scorers` on the eval result; `DEFAULT_VFC_CATALOG`. -- `src/openultrasast/cli.py`: `--slice` choice `vfc`. -- `src/openultrasast/pair_gate.py`: comment that vfc is honesty-only; fail condition stays local. -- `tests/test_pair_corpus.py`: vfc load, provenance, CLI, not in `LANGUAGE_MANIFESTS`, pair_gate still passes. -- `benchmarks/pairs/datasets.toml`: project pointers. -- `benchmarks/pairs/README.md`: slice list and harvest pointer. - -## System Flows - -```mermaid -sequenceDiagram - participant Op as Operator - participant CLI as pairs command - participant Cat as vfc catalog - participant Eval as pair eval - participant Inv as inventory scan - participant Ov as overlay adjudicate - Op->>CLI: pairs slice vfc - CLI->>Cat: load concatenated catalogs - Cat->>Eval: each vfc pair - Eval->>Inv: vuln tree and fix tree - Eval->>Ov: same trees - Ov-->>Eval: promote or fallback - Inv-->>Eval: inventory sidecar - Eval-->>CLI: outcomes plus scorers - CLI-->>Op: JSON or text scoreboard -``` - -Harvest (maintainer, not CI): recipe → fetch parent blob and fix blob → extract named function → write provenance header → write vuln/fixed files. Catalog rows stay hand-reviewed. - -## Requirements Traceability - -| Requirement | Summary | Components | Interfaces | Flows | -|-------------|---------|------------|------------|-------| -| 1.1, 1.2, 1.3 | vfc slice with three projects | VfcCatalog | catalog load | pairs command | -| 1.4 | reject advisory-only | HarvestTool, VfcCatalog | recipe validation | harvest | -| 2.1–2.4 | provenance and same relpath | ProvenanceHeader, VfcCatalog | catalog fields + file header | harvest | -| 3.1–3.4 | honesty not merge gate | PairGatePolicy, DualScorer | pair_gate, payload | gate | -| 4.1–4.4 | overlay vs inventory | DualScorer | scorers map | pairs command | -| 5.1–5.5 | harvest without clones | HarvestTool | recipes + harvest script | harvest | -| 6.1–6.4 | isolated reviewed functions | VfcCatalog, SeedExcerpts | expected CWE/sink | pair eval | -| 7.1–7.4 | offline testability | VfcTestMatrix | pytest + CLI | extra-free CI | - -## Components and Interfaces - -| Component | Domain/Layer | Intent | Req Coverage | Key Dependencies | Contracts | -|-----------|--------------|--------|--------------|------------------|-----------| -| VfcCatalog | Catalog | Reviewed vfc rows | 1.1, 1.2, 1.3, 2.3, 6.4 | Pair loader P0 | State | -| ProvenanceHeader | Catalog | Auditable origin on disk | 2.1, 2.2, 2.4 | VfcCatalog P0 | State | -| HarvestTool | Maintainer | Extract one function from public blobs | 1.4, 5.1, 5.2, 5.3, 5.4 | urllib P1 | Batch | -| DualScorer | Eval | Overlay primary + inventory sidecar | 4.1, 4.2, 4.3, 4.4, 3.3 | overlay eval P0 | State | -| PairGatePolicy | Eval | Local-only fail | 3.1, 3.2, 3.4, 7.4 | pair_gate P0 | State | -| SeedExcerpts | Catalog | Three isolated functions | 1.2, 6.1, 6.2, 6.3 | VfcCatalog P0 | State | -| VfcTestMatrix | Tests | Offline load and CLI | 7.1, 7.2, 7.3, 5.5 | pytest P0 | State | - -### Catalog - -#### VfcCatalog - -| Field | Detail | -|-------|--------| -| Intent | Concatenated `slice = "vfc"` rows with OpenSSL, Firefox, Chromium seeds | -| Requirements | 1.1, 1.2, 1.3, 2.3, 6.4 | - -**Responsibilities & Constraints** -- Same `[[pair]]` / `[[pair.expected]]` schema as github/sast. -- `load_pair_catalog` appends `benchmarks/pairs/vfc/catalog.toml` when loading the default catalog. -- Names stay out of `LANGUAGE_MANIFESTS`. -- Runnable set stays small; recipes may list more than CI runs. - -**Dependencies** -- Inbound: pair loader: concatenate file (P0) -- Outbound: evaluate_pair: cases (P0) - -**Contracts**: State [x] - -##### State Management -- `DEFAULT_VFC_CATALOG` next to `DEFAULT_SAST_CATALOG`. -- Missing vfc file: do not crash default load if we require the seed to exist; the seed is committed so the file is present. Tests assert it exists. - -#### ProvenanceHeader - -| Field | Detail | -|-------|--------| -| Intent | Human-readable origin on every vendored excerpt | -| Requirements | 2.1, 2.2, 2.4 | - -**Responsibilities & Constraints** -- Header comments include repo, commit, parent, commit URL, CVE, license, function name. -- Catalog row repeats repo, commit, commit_url, cve, license. -- License must be stated (OpenSSL License, MPL-2.0, BSD-3-Clause for the seeds). - -#### SeedExcerpts - -| Field | Detail | -|-------|--------| -| Intent | Isolated functions, same relpath, reviewed labels | -| Requirements | 1.2, 6.1, 6.2, 6.3 | - -Seed pairs (reviewed): - -| Name | Project | CVE | Function | Expected class | Honesty note | -|------|---------|-----|----------|----------------|--------------| -| openssl-cve-2014-0160 | OpenSSL | CVE-2014-0160 | `tls1_process_heartbeat` | CWE-126 / memcpy | memcpy remains after fix → likely LEAK | -| firefox-cve-2020-15667 | Firefox libmar | CVE-2020-15667 | `mar_insert_item` | CWE-119 / memcpy | namelen signedness fix; memcpy remains → likely LEAK | -| chromium-cve-2019-5786 | Chromium Blink | CVE-2019-5786 | `FileReaderLoader::ArrayBufferResult` | CWE-416 UAF | no inventory sink → likely MISS | - -Stubs (SSL types, MarItem, DOMArrayBuffer) are allowed only to keep the changed function intact. - -### Eval - -#### DualScorer - -| Field | Detail | -|-------|--------| -| Intent | Overlay-with-fallback as primary vfc metrics; inventory sidecar | -| Requirements | 4.1, 4.2, 4.3, 4.4, 3.3 | - -**Responsibilities & Constraints** -- `evaluate_pair` uses overlay when `slice in {"sast", "vfc"}`. -- Inventory-only evaluation of vfc cases fills `PairEvalResult.scorers["vfc"]["inventory"]`. -- Overlay metrics also appear as `scorers["vfc"]["overlay"]` (same numbers as `per_slice["vfc"]`). -- `result_payload` includes `scorers`. -- Do not use these figures as a gate. - -**Contracts**: State [x] - -```python -class PairEvalResult: - outcomes: tuple[PairOutcome, ...] - overall: PairCorpusMetrics - per_slice: dict[str, PairCorpusMetrics] - signals: tuple[dict[str, object], ...] - scorers: dict[str, dict[str, PairCorpusMetrics]] -``` - -##### Service Interface -- `evaluate_catalog(cases) -> PairEvalResult` -- Preconditions: each case has existing vuln and fixed files. -- Postconditions: if any case has `slice == "vfc"`, `scorers["vfc"]` has `overlay` and `inventory`. -- Invariants: pair_gate fail reasons mention only local. - -#### PairGatePolicy - -| Field | Detail | -|-------|--------| -| Intent | Keep merge fail on local 100% only | -| Requirements | 3.1, 3.2, 3.4, 7.4 | - -`pair_gate` may still *evaluate* the full catalog for the printed dashboard. It must not add vfc pair_correct to `reasons`. - -### Maintainer - -#### HarvestTool - -| Field | Detail | -|-------|--------| -| Intent | Fetch parent/fix blobs and extract one function | -| Requirements | 1.4, 5.1, 5.2, 5.3, 5.4 | - -**Contracts**: Batch [x] - -##### Batch / Job Contract -- Trigger: maintainer runs harvest with a recipe name. CI does not. -- Input: `recipes.toml` row: name, host, repo, parent, commit, path, function, cve, license, relpath. -- Validation: parent and commit required; reject recipes that have only an advisory URL. -- Output: vuln/fixed files with provenance headers. Catalog row is still a human edit. -- Idempotency: overwrite excerpts for that recipe name; do not fetch in tests. - -Function extract: start at the named declarator, copy through balanced braces. Do not clone a repo; HTTP GET of a raw blob or a raw diff. - -Hosts: `raw.githubusercontent.com`, Phabricator `?diff=1`. Not FixFox. - -### Tests - -#### VfcTestMatrix - -| Field | Detail | -|-------|--------| -| Intent | Offline proof the corpus exists and the slice runs | -| Requirements | 7.1, 7.2, 7.3, 5.5 | - -- Catalog load includes `vfc`; files exist; names include openssl, firefox, chromium. -- Provenance headers contain commit URL, CVE, license. -- `ousast pairs --slice vfc --json` exits 0; payload has those names and `scorers.vfc`. -- Pair gate still passes; vfc names not in `LANGUAGE_MANIFESTS`. -- Harvest extract is unit-tested on a local string, not the network. -- Do not assert 100% pair_correct or a Youden floor. - -## Data Models - -### Domain Model - -- **Pair**: isolated vuln snapshot + fixed snapshot + expected CWE/sink + provenance. -- **Slice**: `vfc` honesty set. Not a gate. -- **Scorer**: `overlay` or `inventory`. -- **Recipe**: harvest input, not ground truth until reviewed into the catalog. - -### Logical Data Model - -Catalog row (existing fields plus required provenance for vfc): - -- `name`, `slice = "vfc"`, `language`, `origin`, `license`, `repo`, `commit`, `parent`, `commit_url`, `cve`, `vuln`, `fixed`, `relpath`, `min_recall`, `fix_policy = "silent"` -- `[[pair.expected]]`: `cwe`, `class`, `path`, `rule_id`, `sink`, `evidence` - -`scorers` payload: - -```json -{ - "scorers": { - "vfc": { - "overlay": { "pairs": 3, "pair_correct": 0, "youden": 0.0 }, - "inventory": { "pairs": 3, "pair_correct": 0, "youden": 0.0 } - } - } -} -``` - -Exact counts are not a contract; presence of both objects is. - -## Error Handling - -- Missing vfc catalog file when loading the default catalog: fail tests (seed is required). Loading an explicit `--catalog` path does not auto-append vfc. -- Harvest HTTP failure: non-zero exit, no partial file without a provenance header. -- Advisory-only recipe: reject before fetch. -- Overlay parse_failed / language_unsupported: inventory fallback (existing sast behavior). - -## Testing Strategy - -- Unit: harvest brace-extract on a fixture string; catalog load sees three project names; provenance fields non-empty. -- Integration: `evaluate_catalog(select_slice(..., "vfc"))` returns three outcomes and both scorers; pair_gate still passed. -- CLI: `main(["pairs", "--slice", "vfc", "--json"])` == 0. -- Regression: `LANGUAGE_MANIFESTS` leak check includes vfc names; extra-free suite still green. - -## Security Considerations - -- Vendored excerpts are public historical vuln code with license headers. Do not vendor exploit PoCs (Chromium FileReader exploit HTML is out of scope; only the C++ function). -- Harvest talks only to listed public hosts. No Bugzilla credentials. No FixFox. - -## Supporting References - -Seed commit identifiers are recorded in `research.md` and must appear in the vendored headers at implementation time. diff --git a/.kiro/specs/real-world-vfc-slice/requirements.md b/.kiro/specs/real-world-vfc-slice/requirements.md deleted file mode 100644 index a18adcf..0000000 --- a/.kiro/specs/real-world-vfc-slice/requirements.md +++ /dev/null @@ -1,93 +0,0 @@ -# Requirements Document - -> **Status 2026-10-02:** superseded by `pair-corpus-honesty` (spec.json `superseded_by`); its 12 tasks were done and the vfc pairs live on in that spec's corpus. Kept as decision history; nothing in src, tests or CI points here. - -## Introduction - -AppSec engineers cannot tell whether OpenUltraSAST improved on real product code. The labeled pair slices today are in-tree fixtures (`local`), small GitHub/SVEN functions (`github`), and textbook OWASP/Juliet files (`sast`). Operators care whether the harness fires on a real buggy snapshot of OpenSSL, Firefox, or Chromium and stays quiet after the patch. This spec adds a reviewed, isolated-function **vfc** slice drawn from public vulnerability-fixing commits in those projects, scored as an honesty dashboard (not a merge gate), so the tool can be evolved against labeled product code instead of cheat-sheet recall. - -## Boundary Context - -- In scope: A new pair slice of reviewed isolated function pairs from OpenSSL, Firefox (gecko-dev / mozilla-central / mozilla-firefox/firefox), and Chromium; provenance on every vendored file; overlay-versus-inventory scoreboard on that slice; a harvest procedure that does not clone megarepos; offline runnable catalog; testability of catalog load and slice evaluation. -- Out of scope: Cloning mozilla-central or Chromium into the repository or CI; FixFox (embargoed until 2030); live Bugzilla scraping; making overlay or vfc Youden a merge or smoke gate; buffer-size / CWE-121 semantics; inter-file taint; parser packaging; changing pattern inventory rules so the seed pairs pass. -- Adjacent expectations: Existing pair evaluation still scores `local` as the 100% CI gate and `github` / `sast` as honesty dashboards. Overlay dispositions, taint, and the stage-1 detection gate stay owned by their current specs. Dataset pointers may list Firefox/Chromium/OpenSSL sources; the dumps themselves are not vendored. - -## Requirements - -### Requirement 1: Seed vfc slice from OpenSSL, Firefox, and Chromium - -**Objective:** As an AppSec engineer, I want a labeled vuln-versus-fix slice from OpenSSL, Firefox, and Chromium, so that I can measure the harness on real product functions instead of only textbook files. - -#### Acceptance Criteria - -1. When an operator requests the vfc pair slice, OpenUltraSAST shall evaluate a catalog of isolated vuln-versus-fix pairs whose slice name is `vfc`. -2. The vfc slice shall include at least one reviewed pair whose origin project is OpenSSL, at least one whose origin project is Firefox, and at least one whose origin project is Chromium. -3. When the default pair catalog is loaded, the system shall include the vfc pairs together with the existing local, github, and sast pairs. -4. If a candidate has only an advisory write-up and no public version-control commit for both the vulnerable and fixed snapshots, then the system shall not accept that candidate as a vfc pair. - -### Requirement 2: Provenance on every vendored pair - -**Objective:** As a maintainer, I want every vfc file to carry repo, commit, identifier, and license, so that labels can be audited and redistributed legally. - -#### Acceptance Criteria - -1. When a vfc pair is vendored, the system shall record the source repository, the fixing commit identifier, a public commit URL, the CVE or equivalent public identifier when one exists, and the license of the original file. -2. When a vfc source file is stored in the catalog, the file shall contain a human-readable provenance header with those same fields. -3. The system shall use the same relative path on the vulnerable snapshot and the fixed snapshot of a pair. -4. If license information cannot be stated for a candidate, then the system shall not vendor that candidate. - -### Requirement 3: Honesty dashboard, not a merge gate - -**Objective:** As a developer merging changes, I want vfc misses and leaks to be visible without failing CI, so that real-project honesty cannot be gamed by blocking the merge. - -#### Acceptance Criteria - -1. When the pair efficiency gate runs, the system shall not fail because a vfc pair is a miss or a leak. -2. The system shall not add vfc pair names to the stage-1 detection smoke-gate fixture list. -3. When an operator evaluates the vfc slice, the system shall report pair-correct, vuln-side detection, silent-on-fix, and Youden for that slice. -4. The system shall not require the vfc slice to reach a 100% pair-correct rate, and shall not treat overlay or vfc Youden as a merge condition. - -### Requirement 4: Overlay versus inventory scoreboard - -**Objective:** As an engineer evolving adjudication, I want inventory and overlay scores side by side on the vfc slice, so that I can see whether the overlay helped or hurt on real product functions. - -#### Acceptance Criteria - -1. When an operator evaluates the vfc slice, the system shall report inventory scoring and overlay scoring for that slice as separate scoreboard figures. -2. Where overlay cannot adjudicate a vfc file, the overlay scoreboard shall fall back to inventory for that pair rather than recording silence as safety. -3. The system shall not replace the local-slice pair gate with overlay or vfc Youden. -4. When the vfc scoreboard is emitted as machine-readable output, both scorers shall be present so a later run can be compared without re-deriving the split. - -### Requirement 5: Harvest without megarepos or embargoed dumps - -**Objective:** As a maintainer, I want a repeatable way to add reviewed pairs from public commits, so that the catalog can grow without vendoring Firefox, Chromium, or 16 GB VFC dumps. - -#### Acceptance Criteria - -1. The system shall provide a harvest procedure that fetches a named file at a parent commit and a fixing commit from a public host and extracts one function into the vuln and fixed snapshots. -2. When harvest runs, the system shall not require a full clone of mozilla-central, gecko-dev, or Chromium to produce a pair. -3. The system shall not vendor FixFox or other embargoed datasets. -4. DiverseVul-style automatic vulnerable labels shall not be treated as ground truth unless a human review step has accepted the pair. -5. The runnable catalog that CI evaluates shall remain a small reviewed set (on the order of tens of pairs, not thousands). - -### Requirement 6: Isolated, reviewed functions the overlay can be honest about - -**Objective:** As an operator, I want each pair to be a single isolated function with a reviewed label, so that a miss or leak is attributable to the harness rather than to an unlabeled megatree. - -#### Acceptance Criteria - -1. When a vfc pair is evaluated, the scanner shall see only the vulnerable file or only the fixed file, never both in the same tree. -2. Each vfc pair shall isolate the changed function (plus the minimum types or stubs needed to keep that function intact), not an entire project tree. -3. The catalog shall prefer injection, XSS, or path-traversal pairs when a public commit exists, and may include memory-safety pairs as known inventory-only leaks until a later memory-safety spec. -4. A maintainer shall be able to read the expected CWE, sink, and evidence for each seed pair. - -### Requirement 7: Slice is testable offline - -**Objective:** As a maintainer, I want CI to prove the vfc catalog loads and the slice command runs without network access, so that the labeled corpus cannot rot unnoticed. - -#### Acceptance Criteria - -1. When the test suite loads the default pair catalog, it shall observe vfc among the slice names and shall observe that every vfc vuln file and fixed file exists on disk. -2. When an operator runs the pairs command for slice `vfc`, the command shall complete and emit a scoreboard that includes the OpenSSL, Firefox, and Chromium seed pair names. -3. If the semantic extra is absent, then vfc evaluation shall still complete (via inventory fallback) rather than fail the extra-free suite. -4. The extra-free detection smoke gate shall still ignore vfc pair names and vfc Youden. diff --git a/.kiro/specs/real-world-vfc-slice/research.md b/.kiro/specs/real-world-vfc-slice/research.md deleted file mode 100644 index ab4c724..0000000 --- a/.kiro/specs/real-world-vfc-slice/research.md +++ /dev/null @@ -1,106 +0,0 @@ -# Research & Design Decisions - -## Summary - -- Feature: `real-world-vfc-slice` -- Discovery Scope: Extension of the existing pair catalog and pair eval CLI -- Key Findings: - - Pair eval already concatenates `benchmarks/pairs/catalog.toml` with `sast/catalog.toml`. A third file `vfc/catalog.toml` is the smallest seam. - - CLI `--slice` choices are a closed set (`all`, `local`, `github`, `sast`). `vfc` must be added there and in tests that assert slice names. - - `sast` already scores overlay with inventory fallback when nothing is promoted or demoted. `vfc` should use that path as the primary scorer and add a second inventory-only figure for the honesty dashboard. - - `pair_gate` evaluates the full catalog but fails CI only on `local` 100%. Adding vfc pairs does not change the fail condition if names stay out of `LANGUAGE_MANIFESTS`. - - Seed commits with public VCS snapshots exist: OpenSSL Heartbleed (`tls1_process_heartbeat`), Firefox libmar (`mar_insert_item`, CVE-2020-15667), Chromium FileReader (`ArrayBufferResult`, CVE-2019-5786). All three keep a sink-shaped call after the fix, so overlay/inventory may LEAK; that is the dashboard, not a 100% gate. - - mozilla-central / gecko-dev must not be cloned. Phabricator and GitHub raw blobs are enough to extract one function. FixFox stays embargoed. - -## Research Log - -### Existing pair catalog and scoring - -- Context: Brownfield extension; must not invent a second eval engine. -- Sources Consulted: `src/openultrasast/pairs.py`, `cli.py`, `pair_gate.py`, `tests/test_pair_corpus.py`, `benchmarks/pairs/catalog.toml`, `benchmarks/pairs/sast/catalog.toml`, `benchmarks/pairs/README.md` -- Findings: - - `PairCase.slice` is a free string in the catalog loader; the closed set lives only in argparse and tests. - - Overlay pair eval is gated on `case.slice == "sast"`. - - GitHub pairs already use provenance comments plus catalog fields (`repo`, `commit`, `commit_url`, `cve`, `license`). - - Pair names must not appear in `LANGUAGE_MANIFESTS`. -- Implications: Add `vfc` as a concatenated catalog; extend the overlay branch to `sast` **or** `vfc`; keep the gate local-only. - -### Harvestable public commits (OpenSSL, Firefox, Chromium) - -- Context: User asked for those three projects and a labeled corpus that can actually be tested. -- Sources Consulted: - - OpenSSL `96db9023b881d7cd9f379b0c154650d6c108e9a3` (parent `0d7717fc9c83dafab8153cbd5e2180e6e04cc802`), `ssl/t1_lib.c` `tls1_process_heartbeat`, CVE-2014-0160 - - Firefox hg `b79b6cc78248eea7fda10bfb76aa273c19c9fa65` (parent `0c0f777161a9499dd149853ff62d356f75d16c2a`), `modules/libmar/src/mar_read.c` `mar_insert_item`, CVE-2020-15667 / Bug 1653371; Phabricator raw diff - - Chromium `ba9748e78ec7e9c0d594e7edf7b2c07ea2a90449` (parent `9f3fccdba567f30ea39b08f57d8d5b49cd832b4a`), `FileReaderLoader::ArrayBufferResult`, CVE-2019-5786 - - Rejected: CVE-2019-11730 (SOP/file URI policy, not a sink pair); FixFox (embargoed to 2030); MFSA text without a commit -- Findings: - - Heartbleed fix adds a record-length check; `memcpy(bp, pl, payload)` remains. Inventory `c-unsafe-memory-copy` will fire on both sides (LEAK) until CWE-121 semantics exist. - - Firefox fix is `int namelen` → `uint32_t namelen`; `memcpy(item->name, name, namelen + 1)` remains. Same LEAK class. - - Chromium fix copies the ArrayBuffer on partial results (UAF). No memcpy/eval sink; expected MISS on inventory and overlay. Still a labeled pair. - - GitHub raw and Phabricator `?diff=1` return the function text without cloning. -- Implications: Seed three reviewed pairs. Tests assert presence and that the slice **runs**, not that pair_correct is 100%. Memory-safety pairs are allowed as known leaks (Req 6.3). - -### Dataset pointers versus vendored dumps - -- Context: `datasets.toml` already lists CVEfixes, MoreFixes, SVEN, CWE-Bench-Java. Roadmap asked for Firefox/Chromium/OpenSSL pointers. -- Findings: Pointers are metadata only. CI must stay offline on vendored excerpts. -- Implications: Add dataset rows for OpenSSL, gecko-dev / mozilla-firefox/firefox, and Chromium. Do not fetch dumps in tests. - -## Architecture Pattern Evaluation - -| Option | Description | Strengths | Risks / Limitations | Notes | -|--------|-------------|-----------|---------------------|-------| -| Concatenate a `vfc` catalog like `sast` | Third TOML file, same `[[pair]]` schema | Matches existing loader; no new types | CLI choices and tests must be updated | **Selected** | -| Stuff vfc rows into `catalog.toml` | One file | Fewer paths | Mixes local/github with large-project VFCs | Rejected; sast already split | -| New eval package | Separate scorer | Clean isolation | Duplicates pair eval | Rejected | -| Clone megarepos in CI | Scan real trees | Maximum fidelity | Multi-GB, scrubbed commit messages, not isolated | Explicitly out of scope | - -## Design Decisions - -### Decision: Concatenated `vfc` catalog, overlay primary, inventory sidecar - -- Context: Need a labeled corpus that is offline, isolated, and honest about overlay versus regex. -- Alternatives Considered: - 1. Inventory-only like `github`: misses the overlay question the user is evolving. - 2. Overlay-only like `sast` without a sidecar: cannot see whether overlay helped or hurt. - 3. Dual scoreboard with overlay primary and inventory sidecar: **selected**. -- Selected Approach: `slice = "vfc"` uses the overlay pair path (inventory fallback when unadjudicated). `PairEvalResult.scorers["vfc"]` holds both `overlay` and `inventory` metrics. Pair gate still fails only on `local`. -- Rationale: Reuses `sast` overlay machinery; satisfies Req 4 without making Youden a merge gate. -- Trade-offs: Pair gate runtime grows by three small overlay scans. Acceptable. -- Follow-up: Do not retune memcpy rules to force these pairs to PASS. - -### Decision: Maintainer harvest script, vendored seed, no CI fetch - -- Context: Grow the catalog later without cloning mozilla-central. -- Selected Approach: `benchmarks/pairs/vfc/harvest.py` plus `recipes.toml`. CI never runs harvest. Seed files are committed excerpts with provenance headers. -- Rationale: Req 5 and Req 7 (offline testability). -- Trade-offs: Harvest needs network when a maintainer grows the set; tests mock extraction locally. - -### Decision: Public hg counts as a version-control commit - -- Context: Firefox canonical history is hg.mozilla.org; gecko-dev git SHAs are not the hg SHAs; GitHub 422s on hg identifiers. -- Selected Approach: Record the hg revision and Phabricator/hg URL. Dataset pointers still list gecko-dev and mozilla-firefox/firefox. -- Rationale: Brief forbids MFSA-without-commit, not hg. The MAR fix is a public VCS snapshot. -- Follow-up: If a git SHA is later mapped, add it as an extra catalog field without changing the excerpt. - -## Synthesis - -- Generalization: `vfc` is another honesty slice, not a new product. Overlay-versus-inventory is a report field, not a new engine. -- Build vs adopt: Adopt pair catalog schema, overlay eval, provenance comment style. Build only harvest extract + dual scoreboard payload. -- Simplification: No Joern, no megarepo, no 90% gate, no rule changes to chase Youden on three memory/UAF pairs. - -## Risks & Mitigations - -- Overlay/inventory LEAK on Heartbleed and MAR memcpy-after-fix; document as known; tests must not assert PASS. -- Chromium UAF is a MISS; still required so the corpus includes more than memcpy. -- `pair_gate` currently scans the full catalog; three extra pairs are small. If the catalog grows, gate should keep evaluating local-only for the fail condition (already true) and may later skip overlay slices for speed. -- License: OpenSSL (OpenSSL License / SSLeay), Firefox MPL-2.0, Chromium BSD-3-Clause; headers must state them. -- gecko-dev is archived; harvest should accept hg.mozilla.org and github.com/chromium/chromium / openssl/openssl as hosts. - -## References - -- [OpenSSL Heartbleed fix](https://github.com/openssl/openssl/commit/96db9023b881d7cd9f379b0c154650d6c108e9a3): CVE-2014-0160 -- [Firefox MAR heap overflow](https://bugzilla.mozilla.org/show_bug.cgi?id=1653371): CVE-2020-15667, hg `b79b6cc78248` -- [Chromium FileReader UAF](https://github.com/chromium/chromium/commit/ba9748e78ec7e9c0d594e7edf7b2c07ea2a90449): CVE-2019-5786 -- `.kiro/specs/real-world-vfc-slice/brief.md` -- `.kiro/steering/roadmap.md` diff --git a/.kiro/specs/real-world-vfc-slice/spec.json b/.kiro/specs/real-world-vfc-slice/spec.json deleted file mode 100644 index 55c1a44..0000000 --- a/.kiro/specs/real-world-vfc-slice/spec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "feature_name": "real-world-vfc-slice", - "created_at": "2026-09-04T13:19:17Z", - "updated_at": "2026-09-04T16:12:17Z", - "language": "en", - "phase": "implementation", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true, - "closed": true, - "closed_at": "2026-09-04T16:12:17Z", - "superseded_by": "pair-corpus-honesty" -} diff --git a/.kiro/specs/real-world-vfc-slice/tasks.md b/.kiro/specs/real-world-vfc-slice/tasks.md deleted file mode 100644 index e7820cd..0000000 --- a/.kiro/specs/real-world-vfc-slice/tasks.md +++ /dev/null @@ -1,79 +0,0 @@ -# Implementation Plan - -- [x] 1. Foundation: vfc catalog load and slice command -- [x] 1.1 Load a vfc catalog and accept slice vfc on the pairs command - - Default pair catalog load includes a vfc catalog file the same way it already includes the sast catalog. - - The pairs command accepts slice `vfc` in addition to all, local, github, and sast. - - Dataset pointers name OpenSSL, Firefox/gecko-dev, and Chromium without vendoring those trees. - - An empty or seed-ready vfc catalog file exists so load does not skip the slice by omission. - - Observable: `ousast pairs --slice vfc --json` exits 0 and the loaded slice set can contain `vfc`. - - _Requirements: 1.1, 1.3, 7.2_ - - _Boundary: VfcCatalog_ - -- [x] 2. Core: reviewed seed pairs -- [x] 2.1 Vendor the OpenSSL Heartbleed function pair - - Isolated `tls1_process_heartbeat` before and after the public Heartbleed fix, same relative path. - - Provenance header and catalog row include repo, commit, parent, public commit URL, CVE-2014-0160, and license. - - Expected CWE/sink/evidence are stated; memcpy remaining after the fix is an accepted honesty leak, not a reason to change inventory rules. - - Observable: catalog load yields a vfc pair whose name identifies OpenSSL and whose vuln and fixed files exist. - - _Requirements: 1.2, 1.4, 2.1, 2.2, 2.3, 2.4, 6.1, 6.2, 6.3, 6.4_ - - _Boundary: SeedExcerpts_ - - _Depends: 1.1_ - -- [x] 2.2 Vendor the Firefox libmar function pair - - Isolated `mar_insert_item` before and after the public CVE-2020-15667 commit, same relative path. - - Provenance records the mozilla-central revision and public commit URL (hg or Phabricator is acceptable; MFSA-only is not). - - License is MPL-2.0; expected CWE/sink/evidence are stated. - - Observable: catalog load yields a vfc pair whose name identifies Firefox and whose files exist. - - _Requirements: 1.2, 1.4, 2.1, 2.2, 2.3, 2.4, 6.1, 6.2, 6.3, 6.4_ - - _Boundary: SeedExcerpts_ - - _Depends: 1.1_ - -- [x] 2.3 Vendor the Chromium FileReader function pair - - Isolated `ArrayBufferResult` before and after the public CVE-2019-5786 commit, same relative path. - - Provenance header and catalog row include chromium repo, commit, parent, commit URL, CVE, and BSD-3-Clause. - - UAF-without-inventory-sink is an accepted honesty miss; do not vendor exploit HTML. - - Observable: catalog load yields a vfc pair whose name identifies Chromium and whose files exist. - - _Requirements: 1.2, 1.4, 2.1, 2.2, 2.3, 2.4, 6.1, 6.2, 6.3, 6.4_ - - _Boundary: SeedExcerpts_ - - _Depends: 1.1_ - -- [x] 3. Harvest without megarepos -- [x] 3.1 (P) Add a maintainer harvest procedure that extracts one function - - A recipe names parent commit, fix commit, path, function, CVE, and license; advisory-only recipes are rejected. - - Harvest fetches blobs over HTTP and writes provenance-headed excerpts; it does not clone mozilla-central, gecko-dev, or Chromium. - - FixFox and other embargoed dumps are not referenced as fetch sources. - - Function extract is unit-tested on a local string so CI stays offline. - - Observable: harvest extract on a fixture function round-trips the function body; pytest does not perform a live fetch. - - _Requirements: 1.4, 5.1, 5.2, 5.3, 5.4, 5.5, 7.3_ - - _Boundary: HarvestTool_ - - _Depends: 1.1_ - -- [x] 4. Integration: overlay versus inventory scoreboard -- [x] 4.1 Score vfc with overlay primary and inventory sidecar - - vfc pairs use the overlay pair path with inventory fallback when overlay does not adjudicate. - - Machine-readable pairs output includes `scorers.vfc.overlay` and `scorers.vfc.inventory`. - - Primary `per_slice.vfc` matches the overlay scorer. - - Pair gate fail reasons stay local-only; vfc Youden is not a merge condition. - - Observable: `ousast pairs --slice vfc --json` contains both scorer objects and the OpenSSL, Firefox, and Chromium pair names. - - _Requirements: 3.3, 3.4, 4.1, 4.2, 4.3, 4.4, 7.2_ - - _Boundary: DualScorer_ - - _Depends: 1.1, 2.1, 2.2, 2.3_ - -- [x] 5. Validation: honesty gates and offline tests -- [x] 5.1 Prove vfc is testable and is not a smoke or merge gate - - Default catalog load sees slice `vfc` and every vfc file on disk. - - vfc pair names are absent from the stage-1 detection fixture list. - - Pair efficiency gate still passes on local 100% with vfc present. - - Tests do not require 100% vfc pair_correct or a Youden floor. - - Extra-free evaluation still completes. - - Observable: extra-free pytest covering catalog load, gate pass, and vfc JSON is green. - - _Requirements: 3.1, 3.2, 3.4, 5.5, 7.1, 7.3, 7.4_ - - _Boundary: VfcTestMatrix, PairGatePolicy_ - - _Depends: 4.1_ - -## Implementation Notes - -- Seed commits: OpenSSL `96db9023b881d7cd9f379b0c154650d6c108e9a3` (parent `0d7717fc9c83dafab8153cbd5e2180e6e04cc802`); Firefox hg `b79b6cc78248eea7fda10bfb76aa273c19c9fa65` (parent `0c0f777161a9499dd149853ff62d356f75d16c2a`); Chromium `ba9748e78ec7e9c0d594e7edf7b2c07ea2a90449` (parent `9f3fccdba567f30ea39b08f57d8d5b49cd832b4a`). -- Do not retune `c-unsafe-memory-copy` to make Heartbleed or MAR pair_correct. -- Do not clone mozilla-central; do not vendor FixFox. diff --git a/.kiro/specs/three-stage-scan/brief.md b/.kiro/specs/three-stage-scan/brief.md deleted file mode 100644 index 456b935..0000000 --- a/.kiro/specs/three-stage-scan/brief.md +++ /dev/null @@ -1,77 +0,0 @@ -# Brief: three-stage-scan - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Problem - -AppSec engineers and DevOps teams running OpenUltraSAST in CI get a regex static scan scored against cheat-sheet fixtures. They cannot tell what actually exists in a real repository, where complexity concentrates, how to aim tests, or whether a finding is worth a CI failure. The previous HarnessX spec added governance around those regexes (CWE policy, score, shadow status) without making the hunter evolutionary or the evaluation realistic. - -## Current State - -- `ousast scan --mode quick` is a deterministic regex inventory. It works. -- `--mode standard` is labeled a hunter pool but reuses the same regex path unless the optional HarnessX extra *and* a hunter model are both set. That hunter dumps a file into a prompt and scrapes JSON; it has no tools. -- `--mode deep` exits with "not implemented." Sandbox config exists on paper (`network=false`, read-only workspace, memory/pids/timeout). No container is ever started. -- Benchmarks are one-line sinks written to match the regexes. The fixtures themselves comment the misses (`regex blind spot`). The ≥90% recall gate is circular. -- CWE policy, project score, shadow status, traces, and SARIF already exist and should not be redesigned. - -## Desired Outcome - -A scan answers three operator questions, in order: - -1. What exists? A static inventory of files, surfaces, and pattern/SARIF hits, honest about being inventory, not verified bugs. -2. Where are problems most likely, and how do I tune tests? A complexity map of hotspots plus test-gap hints a DevOps team can act on. -3. Is this worth fixing? Isolated regression against the top hotspots. Triggerable and reachable → worth a CI fail. Not triggerable → do not fail the build. - -Stages 2–3 use a tool-calling hunter (read / grep / callers) and a Docker-isolated runner. Evaluation uses split-sink and mutation corpora, not cheat sheets. Missing Docker or a model degrades visibly; stage 1 still runs. - -## Approach - -**Map-then-prove on the existing modes.** Keep `--mode quick|standard|deep`: - -- `quick` → stage 1 only -- `standard` → stages 1 + 2 -- `deep` → stages 1 + 2 + 3 - -Do not require the HarnessX extra for the LLM path. Use the existing stdlib OpenRouter client when a hunter model is configured. HarnessX remains an optional alternative runtime. - -Do not add a Python Docker SDK. Drive the Docker CLI via subprocess with the threat-model flags (no network, read-only source, tmpfs scratch, memory/pids/timeout, cap-drop). Core `dependencies = []` stays. - -Complexity map is a new artifact built from existing ranking, entry points, finding density, nesting/size, and test-file presence, not a rewrite of `RankingScore`. Stage 3 writes a worth-fixing verdict that feeds back into a complexity ledger (evolutionary), without editing detection pattern text. - -## Scope - -- In: three-stage scan contract; complexity map + test-tuning hints; Docker-isolated regression and worth-fixing verdicts; tool-calling hunter on hotspots; split-sink/mutation evaluation; CI mapping (PR = stage 1, nightly = stage 2/3); visible degradation; evolutionary feedback from regress → map. -- Out: rewriting CWE policy or project-score formula; auto-authoring regex patterns; full fuzzing campaigns; patch generation/application; network pentest; importing Clearwing. - -## Boundary Candidates - -- Stage orchestrator (which stages a mode runs, artifacts, degradation) -- Complexity map (hotspots, signals, test-gap hints) -- Sandbox runner (Docker CLI, capability probe) -- Regression / worth-fixing (candidate tests, verdicts, ledger feedback) -- Tool-calling hunter (repo-bound tools + model loop for stages 2–3) - -## Out of Boundary - -- PolicyStore / ScoreModel / RulesetStore internals (consume, do not own) -- MetaAgent.evolve regex-status loop (adjacent, do not replace) -- Deep fuzzing, exploit development, auto-patching -- HarnessX packaging / optional extra (may be used, not required) - -## Upstream / Downstream - -- Upstream: existing preprocess, rank, mapping, findings, verification, config, CLI modes, OpenRouter client, threat-model sandbox limits. -- Downstream: CI fail-on `worth-fixing`; later fuzz/patch specs; MCP scan status. - -## Existing Spec Touchpoints - -- Extends: `openrouter-sast-harness` gates `standard_security_harness` and `sandboxed_dynamic_harness` (still unsatisfied). -- Adjacent: `harnessx-self-improving-rulesets` owns CWE policy, score, shadow, optional HarnessX extra. This spec must not reopen those. - -## Constraints - -- Core install stays zero-dependency. Docker CLI and an LLM key are operator-provided runtimes, not Python dependencies. -- Scanned code is untrusted. Stage 3 is the first mode allowed to execute target-derived code, and only inside the sandbox. -- LLM output starts at evidence `suspicion`. It must not be stamped `static_corroboration`. -- Cheat-sheet one-liners may remain as unit fixtures; they must not be the merge gate for stages 2–3. diff --git a/.kiro/specs/three-stage-scan/design.md b/.kiro/specs/three-stage-scan/design.md deleted file mode 100644 index aac358f..0000000 --- a/.kiro/specs/three-stage-scan/design.md +++ /dev/null @@ -1,529 +0,0 @@ -# Design Document - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Overview - -**Purpose.** This feature turns an OpenUltraSAST scan into three operator questions: what exists (static inventory), where problems are most likely and how to aim tests (complexity map), and whether a finding is worth fixing (isolated regression). It is a follow-up to the HarnessX governance spec, not a replacement of CWE policy, scoring, or rules-as-data. - -**Users.** DevOps engineers get a cheap pull-request inventory and a nightly prove-worth-fixing job. AppSec reviewers get a hotspot map and test-gap hints instead of a regex dump. Maintainers get an evaluation gate that split-sink cases can fail. - -**Impact.** Today `--mode standard` reuses regex findings and `--mode deep` exits. After this spec, `quick` remains stage 1, `standard` builds a complexity map and may run a tool-using hunter, and `deep` runs a Docker-isolated regression that is the only path allowed to execute target-derived code. - -### Goals - -- Map existing `--mode quick|standard|deep` onto stages 1 / 1+2 / 1+2+3 with explicit manifest accounting. -- Keep stage 1 zero-dependency, no model, no target execution. -- Emit a complexity map and test-tuning hints that do not collapse to pattern-hit count. -- Prove worth-fixing inside an isolated sandbox; fail CI on `worth_fixing` only when asked. -- Hunter tools follow dataflow; hunter output is `suspicion`. -- Stage 2/3 quality is gated on split-sink and mutation corpora. - -### Non-Goals - -- Rewriting PolicyStore, ScoreModel, or detection pattern text. -- Requiring the HarnessX extra for any stage. -- Full fuzzing, exploit development, or auto-patching. -- Importing Clearwing. -- A second CLI family (`--stage`). - -## Boundary Commitments - -### This Spec Owns - -- Stage plan: which stages a mode runs, skip/degrade accounting, `stages_requested` / `stages_completed` in the manifest. -- Complexity map: `Hotspot` records, signals, test-gap hints, `complexity_map.json`. -- Complexity ledger: overlay of hotspot scores from regression verdicts. -- Sandbox: capability probe and Docker CLI runner that never mounts the host Docker socket. -- Regression / worth-fixing: candidate selection, language recipes, verdicts, `--fail-on worth-fixing`. -- Tool hunter: repo-bound read / grep / callers loop for stages 2–3, evidence honesty. -- Split-sink evaluation: corpora and the "beats pattern-hit ranking" gate. - -### Out of Boundary - -- CWE TSV, `resolve_severity`, project-score formula, rule `enabled|shadow|disabled`. -- `MetaAgent.evolve` pattern-status loop. -- Semgrep/CodeQL execution (ingest of operator-supplied SARIF remains as today). -- Patch generation and repository mutation. -- Sanitizer-image matrix and campaign fuzzing. -- MCP docker/shell tools, and exposing `deep` on MCP (CLI/CI only). -- Trail of Bits `skills.STAGES` (`map`/`hunt`/`verify`/`fix`). That `map` is mapping-discipline skill routing, not the complexity-map scan stage. - -### Allowed Dependencies - -- Existing preprocess, rank, mapping, findings, verification, reports, config, CLI, OpenRouter client, HarnessRuntime events. -- Optional HarnessX extra as an alternate hunter host, not a requirement. -- Operator-provided Docker CLI and LLM credential. -- Stdlib only for new core modules (`subprocess` for Docker, `urllib` already used by OpenRouter). - -### Revalidation Triggers - -- Changing the mode→stage mapping or `--fail-on` vocabulary. -- Changing hotspot identity keys (breaks the complexity ledger). -- Changing sandbox flags or mounting the Docker socket. -- Stamping hunter output above `suspicion`. -- Using cheat-sheet recall as the stage 2/3 merge gate. - -## Architecture - -### Existing Architecture Analysis - -| Existing element | Constraint | -|---|---| -| `cli._run_scan` modes | Reuse names; stop exiting on `deep` | -| `quick_scan_findings` | Stage 1 detector; not the stage 2 hunter | -| `rank.heuristic_rank` | File budget; not the complexity map | -| `mapping.analyze_entry_points` | Input to map signals and caller hints | -| `hunter._run_hunter_task` | Must stop being a regex alias when a model is set | -| `OpenRouterChatClient` | Default hunter transport | -| `SandboxConfig` | Limits already match the threat model; need a runner | -| `verify_finding` | Keep as structural review gate; do not treat it as worth-fixing | -| `gate.py` 90% recall | Stage 1 only after this spec | - -### Architecture Pattern & Boundary Map - -**Selected pattern.** Ordered pipeline: **inventory drives the map; the map drives regression; regression overlays the map.** Execution never mutates governance data. Target code runs only in the sandbox. - -```mermaid -flowchart TB - subgraph S1["STAGE 1 inventory"] - PRE[preprocess and mapping] - RANK[file rank] - INV[pattern and SARIF inventory] - end - subgraph S2["STAGE 2 map"] - SIG[complexity signals] - MAP[hotspot map] - HINT[test-gap hints] - HUNT[tool hunter optional] - end - subgraph S3["STAGE 3 prove"] - CAND[regress candidates] - BOX[Docker CLI sandbox] - VERD[worth-fixing verdicts] - end - LED[complexity ledger] - CI[CI fail-on] - PRE --> RANK --> INV - INV --> SIG - RANK --> SIG - LED --> SIG - SIG --> MAP --> HINT - MAP --> HUNT - HUNT --> MAP - MAP --> CAND - INV --> CAND - CAND --> BOX --> VERD - VERD --> LED - VERD --> CI - INV --> CI -``` - -**Dependency direction:** config/types → preprocess/rank/mapping/findings → complexity map → hunter tools → sandbox → regress → CLI/report/gate. - -### Technology Stack - -| Layer | Choice | Role | Notes | -|-------|--------|------|-------| -| CLI | existing `ousast scan --mode` | Stage selection | Add `--fail-on worth-fixing` | -| Hunter transport | `OpenRouterChatClient` | Tool loop | Stdlib; HarnessX optional | -| Sandbox | Docker CLI via `subprocess` | Isolated execution | No Python docker SDK | -| Map / ledger | JSON artifacts | Attention + evolution | Same overlay style as `rule_policy.json` | -| Evaluation | split-sink fixtures | Stage 2/3 gate | Smoke fixtures stay stage 1 | - -## File Structure Plan - -### Directory Structure - -``` -src/openultrasast/ -├── stages.py # Stage enum, mode map, StagePlan, degradation helpers -├── complexity/ -│ ├── __init__.py -│ ├── signals.py # per-file/function signals from targets, findings, tests -│ ├── map.py # build_complexity_map, Hotspot, ComplexityMap -│ ├── hints.py # test-gap hints, closed test-kind set -│ └── ledger.py # load/apply/write complexity_ledger.json -├── hunter_tools.py # read_file, grep_repo, find_refs; path clamp -├── tool_hunter.py # bounded tool loop over a chat client -├── sandbox/ -│ ├── __init__.py -│ ├── probe.py # docker CLI present and usable -│ └── runner.py # DockerCliRunner, SandboxRunner protocol, FakeSandboxRunner -└── regress/ - ├── __init__.py - ├── candidate.py # select and render candidates - ├── recipes.py # language command recipes - └── verdict.py # run candidates, worth-fixing gate - -benchmarks/fixtures/ -├── split-sink-python/ -├── split-sink-javascript/ -├── split-sink-java/ -└── split-sink-c/ -benchmarks/manifests/split-sink-*.toml -``` - -### Modified Files - -- `cli.py`, run stage 2/3; stop exiting on `deep`; `--fail-on worth-fixing`; write new artifacts. -- `reports.py`, `scan_exit_code` grows `worth-fixing`; report sections for inventory, map, and worth-fixing. -- `config.py`, `ComplexityConfig` (top_k, max_hunter_hotspots) and `RegressConfig` (max_candidates, image pins); **consume** existing `SandboxConfig` (do not add a second sandbox TOML). -- `hunter.py`, schedule remains; execution delegates to `tool_hunter` when a model is set, else inventory-only. -- `verification.py` / reports, inventory vs worth-fixing labeling; hunter findings stay `suspicion`. -- `gate.py`, stage-1 smoke gate unchanged; add split-sink map gate. -- `reports.py`, map and verdict sections in markdown/manifest. -- `.github/workflows/ci.yml`, PR stays quick; optional nightly deep when Docker is available. - -## System Flows - -### Mode to stages - -```mermaid -flowchart LR - Q[quick] --> A[stage 1] - S[standard] --> A - S --> B[stage 2] - D[deep] --> A - D --> B - D --> C[stage 3] -``` - -If stage 2 has no model, the heuristic map still writes. If stage 3 has no Docker, stages 1–2 remain and `sandbox_unavailable` is recorded. - -### Stage 3 worth-fixing - -```mermaid -sequenceDiagram - participant M as ComplexityMap - participant R as RegressionRunner - participant S as DockerCliRunner - participant G as WorthFixingGate - M->>R: top hotspots plus severe inventory - R->>R: render snippet and safety check - alt safety check fails - R->>G: inconclusive - else sandbox missing - R->>G: inconclusive sandbox_unavailable - else recipe missing - R->>G: inconclusive - else runs - R->>S: docker run read-only source - S->>R: exit code stdout stderr - R->>G: triggerable or not_triggerable or already_covered - end - G->>M: ledger overlay -``` - -## Requirements Traceability - -| Requirement | Summary | Components | Flows | -|-------------|---------|------------|-------| -| 1.1–1.6 | Mode maps to stages; manifest accounting; degrade forward | `stages.py`, `cli.py` | Mode to stages | -| 2.1–2.5 | Stage 1 inventory, no execute, no model | existing preprocess/rank/findings | Stage 1 | -| 3.1–3.5 | Complexity map independent of regex count | `complexity/map.py`, `signals.py` | Stage 2 | -| 4.1–4.4 | Test-gap hints, no repo writes | `complexity/hints.py` | Stage 2 | -| 5.1–5.8 | Sandbox regression, worth-fixing, fail-on | `sandbox/*`, `regress/*` | Stage 3 | -| 6.1–6.6 | Tool hunter, path clamp, suspicion | `hunter_tools.py`, `tool_hunter.py` | Stage 2/3 | -| 7.1–7.5 | Ledger overlay, cannot hide sev-5 reachable | `complexity/ledger.py` | Stage 3 → map | -| 8.1–8.6 | Split-sink / mutation gate | fixtures, `gate.py` | Evaluation | -| 9.1–9.6 | PR cheap, nightly bounded, visible skip | `cli.py`, CI workflow | Degradation | - -## Components and Interfaces - -| Component | Domain | Intent | Req Coverage | Dependencies | Contracts | -|-----------|--------|--------|--------------|--------------|-----------| -| StagePlan | Orchestration | Mode → stages, skip reasons | 1.1–1.6, 9.3–9.6 | config | State | -| ComplexityMapBuilder | Map | Hotspots from signals + ledger | 3.1–3.5, 7.1–7.3 | preprocess, rank, findings | Batch, State | -| TestTuningAdvisor | Map | Test-gap hints | 4.1–4.4 | ComplexityMap | Batch | -| HunterTools | Hunter | Read/grep/refs inside repo | 6.1–6.2 | mapping | Service | -| ToolHunter | Hunter | Bounded tool loop | 6.1–6.5 | OpenRouter, HunterTools | Service | -| SandboxProbe | Sandbox | Docker usable? | 5.2, 9.3 | docker CLI | Service | -| DockerCliRunner | Sandbox | Isolated `docker run` | 5.2, 5.8 | docker CLI | Service | -| RegressionRunner | Regress | Candidates, recipes, verdicts | 5.1–5.6 | Sandbox, map | Batch | -| WorthFixingGate | Regress | CI fail-on | 5.4–5.7, 7.4 | RegressionRunner, policy | Batch | - -### Orchestration - -#### StagePlan - -| Field | Detail | -|-------|--------| -| Intent | Declare which stages run and why a stage was skipped | -| Requirements | 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 9.3, 9.4, 9.6 | - -```python -from enum import StrEnum - -class Stage(StrEnum): - STATIC = "static" - MAP = "map" - REGRESS = "regress" - -MODE_STAGES = { - "quick": (Stage.STATIC,), - "standard": (Stage.STATIC, Stage.MAP), - "deep": (Stage.STATIC, Stage.MAP, Stage.REGRESS), -} - -@dataclass(frozen=True) -class StagePlan: - mode: str - requested: tuple[Stage, ...] - completed: tuple[Stage, ...] - skipped: tuple[dict[str, str], ...] # stage, reason -``` - -- **Invariants:** `quick` never includes `MAP` or `REGRESS`. Skipped stages appear in `runtime.state["degradations"]` with the same shape as existing capability fallbacks. -- **Naming:** `Stage.MAP` is the complexity-map scan stage. It is unrelated to `skills.STAGES` value `"map"` (Semgrep/CodeQL skill routing). Do not merge the two enumerations. - -### Complexity map - -#### ComplexityMapBuilder - -| Field | Detail | -|-------|--------| -| Intent | Rank likely problem areas without using pattern-hit count as the score | -| Requirements | 3.1, 3.2, 3.3, 3.4, 3.5 | - -**Signals** (stdlib, no extra libs): - -- size: loc -- nesting: max indent or brace depth -- surface tags from `FileTarget` -- reachability from entry-point hints -- inventory density as a *feature*, not the score (used in rationale, down-weighted in the formula) -- test presence: sibling `test_*` / `*_test` / `*Test.java` / `*_test.go` under common test roots -- ledger overlay last - -**Hotspot identity:** `path` + `function_name or ""`. Function name from reachability hints or a language-specific def/func regex; empty if unknown. Java/Groovy mapping is tag-only today, so identity is usually the file; that is acceptable for ranking and test-gap hints. - -```python -@dataclass(frozen=True) -class Hotspot: - path: str - function_name: str | None - score: float - band: str # high | medium | low - signals: dict[str, float | int | bool | str] - rationale: str - test_hint: TestHint | None - inventory_finding_ids: tuple[str, ...] - -@dataclass(frozen=True) -class ComplexityMap: - hotspots: tuple[Hotspot, ...] - heuristic_only: bool -``` - -- **Invariant:** sorting files by `len(inventory_finding_ids)` descending must not be identical to map order on the split-sink corpus (enforced by the gate). -- **Postcondition:** write `.openultrasast/runs//complexity_map.json`. - -#### TestTuningAdvisor - -Closed `test_kind` set: `unit | property | sanitizer | http-contract | fuzz-harness`. - -```python -@dataclass(frozen=True) -class TestHint: - path: str - function_name: str | None - gap: str # no_adjacent_test | no_function_reference | no_fuzz_entry | covered - test_kind: str | None - reason: str -``` - -- **Invariant:** `gap == "covered"` ⇒ `test_kind is None`. Never writes into the scanned tree. - -#### Complexity ledger - -Path: `/.openultrasast/calibration/complexity_ledger.json`. - -```json -{ - "parser.c:parse_header": { - "score_delta": 1.5, - "last_verdict": "triggerable", - "round": 3 - } -} -``` - -Apply at map build. `score_delta` cannot drop a reachable policy-severity-5 inventory finding out of the stage-3 candidate set. - -### Hunter - -#### HunterTools - -```python -def clamp_repo_path(root: Path, user_path: str) -> Path: ... -def read_file(root: Path, path: str, *, max_chars: int) -> str: ... -def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, object]]: ... -def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, object]]: ... -``` - -- **Invariant:** any resolved path whose common prefix is not `root` raises `PathEscapesRepo`. -- Grep is Python `re` over enumerated source files (zero-dep). Ripgrep is an optional fast path if present, still clamped. - -#### ToolHunter - -Bounded loop: max steps from the hotspot's hunter budget (reuse tier A/B/C caps). Each model turn may call the three tools or stop with a JSON list of suspicions. - -```python -def run_tool_hunter( - root: Path, - hotspots: list[Hotspot], - *, - client: ChatClient, - model: str, - max_steps: int, -) -> list[StaticFinding]: ... -``` - -- **Postconditions:** each finding `evidence_level == "suspicion"`; `finding_id` prefix `tool-hunter:`. -- **Fallback:** no model → empty hunter findings, `heuristic_only=True` on the map, degradation `hunter_model_unavailable`. - -### Sandbox and regression - -#### SandboxRunner - -```python -class SandboxRunner(Protocol): - def run(self, spec: SandboxJob) -> SandboxResult: ... - -@dataclass(frozen=True) -class SandboxJob: - image: str - command: tuple[str, ...] - repo_root: Path - scratch_files: dict[str, str] # relative to /scratch - timeout_seconds: int - memory_mb: int - pids_limit: int -``` - -`DockerCliRunner` argv (conceptual): `docker run --rm --init --network none --read-only --tmpfs /scratch:rw,exec --memory --pids-limit --cap-drop ALL --security-opt no-new-privileges --user` non-root `--mount type=bind,src=repo,dst=/workspace,ro` image command. - -- **Forbidden:** `-v /var/run/docker.sock`, `--network host`, `--privileged`, writable bind of the source tree. -- `SandboxProbe.available()` runs `docker info` with a short timeout. - -#### RegressionRunner - -Select `min(max_candidates, top_k_hotspots ∪ sev5_reachable_inventory)`. Render a recipe snippet into `scratch_files`. Safety check rejects snippets that mention `socket`, `curl`, `docker.sock`, or writes under `/workspace`. - -Verdicts: - -| Observation | Verdict | -|---|---| -| recipe or sandbox missing, timeout, safety reject | `inconclusive` | -| existing project test covering the hotspot already passes in sandbox | `already_covered` | -| candidate exits non-zero with a crash, assertion, or sanitizer abort | `triggerable` | -| candidate exits 0 | `not_triggerable` | - -`worth_fixing` ⇔ `triggerable` and related finding reachability is `reachable` or `inferred-file-surface`. - -`--fail-on` gains `worth-fixing`. Existing `findings` / `verified` unchanged. - -Recipes MVP: Python (`python /scratch/case.py` with `PYTHONPATH=/workspace`), JavaScript (`node /scratch/case.js`), C (`cc` + run) when the image contains a compiler. Java recipe may return `inconclusive` until added. - -## Data Models - -### Manifest additions - -```json -{ - "stages": { - "requested": ["static", "map", "regress"], - "completed": ["static", "map"], - "skipped": [{"stage": "regress", "reason": "sandbox_unavailable"}] - }, - "complexity": {"hotspot_count": 12, "heuristic_only": true}, - "worth_fixing": {"count": 0, "verdicts": []} -} -``` - -Degradations reuse the existing list: `{ "stage": "regress", "reason": "sandbox_unavailable" }`. - -### Consistency - -- Map artifact and ledger are JSON, resume-safe. -- Hunter suspicions never jump evidence levels in this spec. -- Stage 1 artifacts remain byte-compatible with current `findings.json` schema; new fields are additive. - -## Error Handling - -| Error | Response | -|---|---| -| `deep` requested, Docker missing | Complete 1–2, skip 3, degradation, exit 0 unless other fail-on hits | -| Model missing on `standard` | Heuristic map, no hunter findings, degradation | -| Path escape in a tool | Tool error to the model; do not throw the scan | -| Snippet fails safety check | `inconclusive`, do not run | -| Docker run timeout | kill container (`--rm`), `inconclusive` | -| Ledger would hide sev-5 reachable | Candidate still selected | - -## Testing Strategy - -### Unit - -- Mode→stage mapping and skip accounting. -- Path clamp rejects `../`. -- Map order on a fixture where regex hits a trivial helper and the real bug is a split-sink parser, parser ranks higher. -- Covered hotspot yields `gap=covered` and no test kind. -- Safety check rejects docker.sock and network snippets. -- Ledger boost/demote; sev-5 reachable remains a candidate. - -### Integration - -- Fake sandbox: `triggerable` / `not_triggerable` / timeout mapped to verdicts. -- Tool hunter with a scripted chat client that must call `grep_repo` before concluding on a split-sink file; fail if it only returns JSON with no tool call. -- `--fail-on worth-fixing` exits non-zero only when a `worth_fixing` verdict exists. - -### Evaluation / CI - -- Stage 1: existing `python -m openultrasast.gate` on smoke corpora. -- Stage 2: split-sink manifests; true vulnerable function in the high band; map order ≠ pattern-hit order. -- Stage 3: `@pytest.mark.docker`; skip if probe fails; at least one planted split-sink that stage 1 missed is `triggerable`. -- PR workflow stays `quick`. Nightly/deep is a separate job with Docker. - -### Anti-patterns this spec forbids in tests - -- One-line `eval(request.args)` as the only stage-2 proof. -- Asserting hunter findings equal regex findings. -- Calling the real Docker daemon in the default `pytest` job. - -## Security Considerations - -- Scanned code is untrusted. Stage 1–2 read it as text. Stage 3 executes only in Docker with network off, source read-only, no docker.sock, dropped caps, no-new-privileges, non-root user. -- Hunter tools cannot read outside the repo. -- Regression snippets are data, not host shell. They go to `/scratch` in the container. -- Secrets redaction already applies to traces and markdown; keep it on for map rationales that quote source. - -## Performance & Scalability - -- Map is O(files) plus one pass of findings. No extra model required. -- Tool hunter runs only on `max_hunter_hotspots` (default 8). -- Stage 3 `max_candidates` default 5; timeout from `SandboxConfig.timeout_seconds`. -- Large-repo CI: PR = stage 1; nightly = stage 2 always, stage 3 if probe passes. - -## Migration Strategy - -```mermaid -flowchart LR - P0[StagePlan plus manifest] --> P1[Heuristic map and hints] - P1 --> P2[Split-sink gate] - P2 --> P3[Tool hunter] - P3 --> P4[Docker runner plus recipes] - P4 --> P5[Ledger and fail-on worth-fixing] -``` - -| Phase | Ships | Merge gate | Rollback | -|---|---|---|---| -| 0 StagePlan | `deep` no longer exits; skips 2/3 with degradations | existing tests | revert CLI | -| 1 Heuristic map | `complexity_map.json` on `standard` | unit map tests | feature: empty map writer | -| 2 Split-sink gate | new fixtures; map ≠ hit-count | new gate job | gate off | -| 3 Tool hunter | OpenRouter loop, suspicion | scripted-client test | model unset | -| 4 Sandbox | Docker CLI + Python/JS recipes | docker-marked tests skip-ok | probe skip | -| 5 Worth-fixing | ledger + fail-on | integration on fake runner | flag unused | - -`deep` changing from hard-exit to skip-or-run is intentional and documented. No artifact schema is removed. diff --git a/.kiro/specs/three-stage-scan/requirements.md b/.kiro/specs/three-stage-scan/requirements.md deleted file mode 100644 index acb04a2..0000000 --- a/.kiro/specs/three-stage-scan/requirements.md +++ /dev/null @@ -1,134 +0,0 @@ -# Requirements Document - -> **Status 2026-10-02:** retired (see the note below); all 31 tasks were done before HarnessX was removed. Kept as decision history; nothing in src, tests or CI points here. - -> **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax -> (`ai-service-plane`), and evolutionary proposals come from the plane memory. History below is unchanged. - -## Introduction - -OpenUltraSAST today answers one question well: which regexes fired. It does not answer the three questions a rugged DevOps CI workflow actually asks: what exists in this repository, where problems are most likely and how to aim tests, and whether a finding is worth fixing. This feature turns the scan into those three stages. Stage 1 is a static inventory of what exists. Stage 2 is a complexity map of likely problem areas plus test-tuning hints. Stage 3 is isolated regression that proves triggerability before CI treats a finding as blocking. - -The work is a follow-up to the HarnessX ruleset-governance spec, not a replacement of it. CWE policy, project score, and rule-status shadowing stay in place. This spec owns the stage contract, the complexity map, the isolated regression verdict, the tool-using hunter on stages 2–3, and an evaluation corpus that is not a cheat sheet. - -## Boundary Context - -- In scope: a three-stage scan contract mapped onto existing scan depths; a static inventory that reports what exists without claiming verification; a complexity map of hotspots with test-tuning hints; isolated regression that produces a worth-fixing verdict; a tool-using hunter confined to stages 2–3; evaluation against split-sink and mutation corpora; visible degradation when the sandbox or a model is unavailable; evolutionary feedback from regression verdicts back into the complexity map. -- Out of scope: rewriting CWE-to-severity policy or the project-score formula; authoring or rewriting detection pattern text; full fuzzing campaigns; generating or applying patches; network pentesting; importing Clearwing. -- Adjacent expectations: stage 1 continues to run in the zero-dependency default install with no model and no sandbox. CWE policy, scoring, traces, and SARIF remain owned by the existing governance spec. The operator supplies an isolated-sandbox runtime (Docker CLI) for stage 3 and an LLM credential for the tool-using hunter; neither is a packaged library dependency of the core install. - -## Requirements - -### Requirement 1: Three-Stage Scan Contract - -**Objective:** As a DevOps engineer, I want a scan to run as three ordered stages that map onto the existing scan depths, so that CI can choose inventory-only, map-and-tune, or prove-worth-fixing without a new command family. - -#### Acceptance Criteria - -1. When a scan runs at depth `quick`, the Scan Orchestrator shall execute stage 1 only and shall not start a model call or execute target-derived code. -2. When a scan runs at depth `standard`, the Scan Orchestrator shall execute stage 1 then stage 2. -3. When a scan runs at depth `deep`, the Scan Orchestrator shall execute stage 1 then stage 2 then stage 3. -4. The Scan Orchestrator shall record the stages requested, the stages completed, and any skipped stage with a reason in the scan manifest. -5. If a later stage cannot run, the Scan Orchestrator shall complete earlier stages, record a degradation, and shall not discard those earlier artifacts. -6. The Scan Orchestrator shall keep the existing scan command and depth names so existing CI invocations continue to work. - -### Requirement 2: Stage 1 Static Inventory - -**Objective:** As a security reviewer, I want a static inventory of what exists in the repository, so that I can see files, surfaces, and pattern hits without treating them as verified vulnerabilities. - -#### Acceptance Criteria - -1. When stage 1 completes, the Static Inventory shall emit a repository snapshot, file targets with language and surface tags, entry-point reachability hints, ranked files, and static findings from the existing pattern ruleset and any provided static-analyzer input. -2. The Static Inventory shall label static pattern hits as inventory, not as verified findings, in the report and in the finding evidence level. -3. While no static-analyzer input is provided, the Static Inventory shall still complete using the built-in pattern ruleset and heuristics. -4. If an enabled rule's CWE does not resolve in policy, the Scan Runtime shall fail loud before producing inventory findings (existing startup invariant, preserved). -5. The Static Inventory shall not execute target-derived code and shall not require a model. - -### Requirement 3: Stage 2 Complexity Map - -**Objective:** As an AppSec engineer, I want a complexity map of where problems are most likely, so that review and testing attention go to real hotspots instead of every regex hit. - -#### Acceptance Criteria - -1. When stage 2 runs, the Complexity Map shall produce an ordered list of hotspots, each identifying a file and, when known, a function, with a complexity score and a rationale built from size, nesting, surface tags, reachability, existing inventory-hit density, and test-presence gaps. -2. The Complexity Map shall rank hotspots independently of a single regex match: a file with no pattern hit may still be a hotspot, and a pattern hit in a trivial helper may rank below a complex reachable parser. -3. When stage 2 completes, the Complexity Map shall write a map artifact that the report and later stages consume, including the scoring signals used for each hotspot. -4. While a hunter model is unavailable, the Complexity Map shall still produce the heuristic map and shall record that the hunter did not refine it. -5. The Complexity Map shall not mark a hotspot as verified or worth fixing. - -### Requirement 4: Test-Tuning Hints - -**Objective:** As a developer owning CI tests, I want the complexity map to tell me how to tune tests, so that I add coverage where it reduces real risk instead of mirroring cheat-sheet sinks. - -#### Acceptance Criteria - -1. When the Complexity Map emits a hotspot, the Test Tuning Advisor shall attach a test-gap hint that names the missing coverage (no adjacent test file, no test referencing the function, or no sanitizer/fuzz entry for a fuzzable surface). -2. The Test Tuning Advisor shall recommend a test kind from a closed set: unit, property, sanitizer, http-contract, or fuzz-harness, with a one-line reason tied to the hotspot's signals. -3. The Test Tuning Advisor shall not write test files into the scanned repository. -4. When a hotspot already has covering tests, the Test Tuning Advisor shall say so and shall not recommend a duplicate test kind. - -### Requirement 5: Stage 3 Isolated Regression and Worth-Fixing - -**Objective:** As a CI owner, I want isolated regression that answers whether a finding is worth fixing, so that the build fails only on triggerable, reachable issues. - -#### Acceptance Criteria - -1. When stage 3 runs, the Regression Runner shall select a bounded set of top hotspots and/or inventory findings and shall attempt a minimal regression or proof-of-concept inside an isolated analysis sandbox. -2. The isolated analysis sandbox shall run with network disabled, the source tree mounted read-only, a writable scratch area, and configured memory, process, and timeout limits. -3. The Regression Runner shall emit one of four verdicts per candidate: `triggerable`, `not_triggerable`, `already_covered`, or `inconclusive`. -4. When a candidate is `triggerable` and the related finding is reachable, the Worth-Fixing Gate shall mark it `worth_fixing`. -5. When a candidate is `not_triggerable` or `already_covered`, the Worth-Fixing Gate shall mark it `not_worth_fixing` and shall not fail the build for that candidate. -6. When the verdict is `inconclusive`, the Worth-Fixing Gate shall not fail the build for that candidate and shall record why (timeout, sandbox unavailable, no candidate generated, or setup failed). -7. Where `--fail-on worth-fixing` is selected, the Scan Orchestrator shall fail the scan if and only if at least one `worth_fixing` verdict was produced. -8. The Regression Runner shall never execute target-derived code on the host outside the isolated sandbox. - -### Requirement 6: Tool-Using Hunter on Stages 2 and 3 - -**Objective:** As an OpenUltraSAST maintainer, I want the hunter on later stages to use repository tools instead of dumping a file into a prompt, so that it can follow dataflow across lines and files the way a real review does. - -#### Acceptance Criteria - -1. Where a hunter model is configured, the Hunter Pool shall run a bounded tool-using loop against stage-2 hotspots, with tools limited to reading a file inside the repository, searching repository text, and listing callers or references already known to the scan. -2. The Hunter Pool shall confine every tool path to the scanned repository root and shall reject path arguments that escape that root. -3. When the hunter reports a candidate issue, the Scan Runtime shall record it at evidence level `suspicion` until independent static or regression evidence raises it. -4. While no hunter model is configured, the Hunter Pool shall skip the tool loop, leave the heuristic complexity map in place, and record a degradation. -5. The Hunter Pool shall not treat a model JSON list as static corroboration. -6. When stage 3 runs and a hunter model is configured, the Hunter Pool may propose a regression snippet, but the Regression Runner shall execute that snippet only inside the isolated sandbox after a structural safety check (no network, no writes outside scratch). - -### Requirement 7: Evolutionary Feedback from Regression to Map - -**Objective:** As a maintainer, I want regression verdicts to retune the complexity map, so that the scan gets better at spending attention where issues actually trigger. - -#### Acceptance Criteria - -1. When a hotspot's regression is `triggerable`, the Complexity Map shall raise that hotspot's subsequent score on the same repository. -2. When a hotspot's regression is `not_triggerable`, the Complexity Map shall lower that hotspot's subsequent score without deleting the underlying inventory hit. -3. The Scan Runtime shall persist those adjustments in a loop-owned ledger keyed by repository and hotspot identity, applied at map load the same way rule status overlays apply at ruleset load. -4. If a ledger entry would hide a reachable severity-5 inventory hit from the worth-fixing set, the Worth-Fixing Gate shall still consider that hit. -5. The evolutionary loop shall not edit detection pattern text and shall not edit authoritative CWE severity. - -### Requirement 8: Grounded Evaluation, Not Cheat Sheets - -**Objective:** As a maintainer, I want stage 2 and stage 3 quality gates to use split-sink and mutation cases, so that a regex that only matches textbook one-liners cannot claim success. - -#### Acceptance Criteria - -1. The Benchmark Harness shall include at least one split-sink case per in-scope language family (Python, JavaScript, Java, C/C++) where the source and the sink are on different lines or the payload is assigned before use. -2. The Benchmark Harness shall include at least one mutation of a known sink (equivalent API, intermediate variable, or wrapper) that a same-line pattern does not match. -3. When stage 1 is scored alone, the Benchmark Harness may use existing smoke fixtures, and shall not treat stage-1 recall as proof of stage-2 or stage-3 quality. -4. When stage 2 is scored, the Benchmark Harness shall require the complexity map to rank the true vulnerable function in the top band of hotspots on the split-sink corpus, even when stage 1 missed the sink. -5. When stage 3 is scored and the sandbox is available, the Benchmark Harness shall require a `triggerable` verdict on at least one planted split-sink that stage 1 missed. -6. The CI merge gate for this feature shall fail if stage 2 ranking on the split-sink corpus is no better than ranking files by pattern-hit count alone. - -### Requirement 9: CI Companion Budgets and Degradation - -**Objective:** As a CI owner, I want stage costs and failures to be bounded and visible, so that pull-request jobs stay cheap and nightly jobs can run the later stages without silent fallback. - -#### Acceptance Criteria - -1. While running in pull-request depth `quick`, the CI Pipeline shall complete stage 1 without a sandbox runtime and without a model. -2. Where a nightly or labeled job requests `standard` or `deep`, the CI Pipeline shall run those stages under the configured time, finding-count, and cost budgets. -3. If the sandbox runtime is missing when stage 3 is requested, the Scan Orchestrator shall skip stage 3, record `sandbox_unavailable`, and complete stages 1–2. -4. If a model is missing when stage 2 hunter refinement is requested, the Scan Orchestrator shall keep the heuristic map, record `hunter_model_unavailable`, and shall not invent hunter findings. -5. The Scan Orchestrator shall bound stage 3 to a configured maximum number of regression candidates per scan. -6. When any stage is skipped or truncated, the report shall show the degradation in the same manifest list used by existing capability fallbacks. diff --git a/.kiro/specs/three-stage-scan/research.md b/.kiro/specs/three-stage-scan/research.md deleted file mode 100644 index e4435ea..0000000 --- a/.kiro/specs/three-stage-scan/research.md +++ /dev/null @@ -1,156 +0,0 @@ -# Research & Design Decisions - ---- -**Purpose**: Capture discovery findings and architectural investigation behind the `three-stage-scan` spec. `design.md` carries the committed architecture; this file carries evidence and rejected alternatives. ---- - -## Summary -- Feature: `three-stage-scan` -- Discovery Scope: Extension of the existing scan harness (brownfield), with one new runtime prerequisite (Docker CLI for stage 3) and one new evaluation contract (split-sink / mutation corpora). -- Key Findings: - - Stage 1 already exists as `quick` mode: preprocess, mapping, rank, `quick_scan_findings`, structural verify, score, SARIF. Do not rebuild it. - - Stage 2 is not a hunter dump and is not `RankingScore`. Ranking is per-file attention. A complexity map is per-file-and-function hotspot attention plus test-gap hints. Existing signals (tags, loc, reachability hints, finding density, `has_fuzz_entry_point`) are enough for a heuristic map; the tool-using hunter refines it. - - Stage 3 has config and a threat-model table, but `cli.py` exits on `--mode deep`. No container is ever started. The first execution of untrusted code must be Docker CLI with the documented isolation flags, not a Python SDK and not host `subprocess` of target code. - - The previous HarnessX spec locked LLM use behind an optional extra that CI does not install. This spec uses the existing stdlib OpenRouter client so stage 2 can be evolutionary without that extra. HarnessX remains an optional runtime, not a prerequisite. - - Cheat-sheet fixtures cannot gate stages 2–3. The Python fixture already documents a split-sink miss (`query` built on the prior line). That class of case becomes the quality bar. - -## Research Log - -### Existing scan pipeline and mode mapping -- Context: Whether to add `--stage` or reuse `--mode quick|standard|deep`. -- Sources Consulted: `src/openultrasast/cli.py` (`_run_scan`, mode choices), `README.md` maturity legend, `.kiro/specs/openrouter-sast-harness/requirements.md` (hunter pool, docker sandbox, dynamic analysis), `.kiro/specs/harnessx-self-improving-rulesets/design.md`. -- Findings: - - `quick` is implemented and deterministic. `standard` schedules a hunter pool that calls `quick_scan_findings` unless HarnessX extra + hunter model are set. `deep` raises `SystemExit`. - - OpenRouter spec already defined: quick = static + bounded LLM review; standard = sandboxed tool-calling hunters; deep = harness/fuzz/dynamic. Implementation never reached that. - - Existing CI and README teach `--mode`. A parallel `--stage` flag would fork the contract. -- Implications: Map modes onto stages 1 / 1+2 / 1+2+3. Record `stages_requested` and `stages_completed` in the manifest so the three questions are explicit even if the flag names stay. - -### Ranking vs complexity map -- Context: `rank.py` already scores files. Is stage 2 just ranking? -- Sources Consulted: `src/openultrasast/rank.py` (`heuristic_rank`, `composite_priority` = surface 0.5 + influence 0.2 + reachability 0.3), `preprocess.py` (`FileTarget` tags, loc, `has_fuzz_entry_point`), `mapping.py` (`analyze_entry_points`). -- Findings: - - Ranking is file-level and exists to budget the hunter. It does not name functions, test gaps, or "tune your tests." - - Tags already encode likely problem classes (`memory_unsafe`, `parser`, `network_entry`, `syscall_entry`). Loc ≥ 500 boosts influence. No nesting/branch signal. No test-file presence. - - A hotspot that stage 1 missed (split-sink) can still rank high if the file is a reachable parser. That is the point of stage 2. -- Implications: Keep `RankingScore`. Add `ComplexityMap` / `Hotspot` with function identity when recoverable, test-gap hints, and a ledger overlay. Ranking still feeds hunter budgets; the map feeds humans and stage 3. - -### Hunter: regex, HarnessX dump, or tools -- Context: Previous spec adopted HarnessX "at the hunter loop" but the default hunter is regex and the HarnessX path dumps source into a prompt. -- Sources Consulted: `hunter.py` (`_run_hunter_task` → `quick_scan_findings`), `hunter_harness.py` (`_describe`, `_extract_findings`), `provider/openrouter.py` (`OpenRouterChatClient.complete_json`, stdlib `urllib`), `harness_ext.py` (optional extra). -- Findings: - - OpenRouter client is already zero-extra (stdlib). It is used for optional ranking, not for hunting. - - HarnessX extra is SHA-pinned, heavy, absent in CI. Requiring it repeats the last spec's failure mode. - - Clearwing hunters use `read_source_file`, `grep_source`, `find_callers` with path clamping. That is the evolutionary piece, not MetaAgent regex-status flips. -- Implications: Stage 2/3 hunter is a bounded tool loop over the OpenRouter client (or HarnessX if already wired). Tools are stdlib (read, Python grep, caller search from mapping + text). Path clamp at repo root. Hunter output evidence = `suspicion`. - -### Docker sandbox: CLI vs SDK vs skip -- Context: Threat model already lists sandbox limits. Need a runner that does not invert zero-dep posture. -- Sources Consulted: `docs/threat-model.md` (sandbox table), `config.py` `SandboxConfig`, Clearwing `hunter_sandbox.py` (image build + spawn, Docker SDK), Docker isolation guidance (`docker run --network none --read-only --tmpfs --memory --pids-limit --cap-drop ALL --security-opt no-new-privileges`), initrunner docker-sandbox docs (CLI `docker run --rm --init`, optional `runsc`). -- Findings: - - Python `docker` SDK would be a new core or extra dependency. The operator already has the Docker CLI if they can run containers. - - Subprocess `docker run --rm --init` with bind-mount of the repo read-only and a tmpfs/scratch write area matches the threat model and keeps `dependencies = []`. - - Unit tests must not require a daemon: inject a `SandboxRunner` protocol, ship a fake. Integration tests skip when `docker info` fails. - - Full sanitizer-image builds (Clearwing) are out of scope for this spec. Stage 3 runs language-specific commands on a small pinned image per language family, or `inconclusive` if the recipe is missing. -- Implications: `SandboxProbe` + `DockerCliRunner`. No SDK. gVisor/`runsc` is optional `security-opt` when `docker info` reports it; not required. - -### Evaluation corpora -- Context: User rejected cheat-sheet fixtures. Current gate is ≥90% recall on those fixtures. -- Sources Consulted: `benchmarks/fixtures/python-vulnerable/app.py` (line 77 split-sink comment), `javascript-vulnerable/src/app.js` (prototype pollution comment), `tests/test_detection_benchmarks.py`, `gate.py`. -- Findings: - - Stage 1 smoke fixtures are still useful as unit tests. They must not be the stage 2/3 merge gate. - - A split-sink corpus is small and owned: source and sink on different lines; mutations through wrappers/variables. Success = map ranks the true function in the top band; stage 3 triggers at least one stage-1 miss when Docker is present. - - Comparing map ranking to "sort by pattern-hit count" is an anti-cheat-sheet test: if regex density is the map, the gate fails. -- Implications: New `benchmarks/fixtures/split-sink-*` and a stage-2 ranking gate. Keep existing `openultrasast.gate` for stage 1 only. - -### Seams that already exist and must not be confused -- Context: Brownfield traps for a spec named "map" that also adds Docker. -- Sources Consulted: `skills.py` (`STAGES = ("map", "hunt", "verify", "fix")`), `mcp.py:68` (rejects `deep`), `reports.py:115` (`scan_exit_code`), `mapping.py` (no Java/Groovy AST), `config.py` (`SandboxConfig` parsed, never consumed by `_run_scan`), `rank.py` (`rank_targets_with_model` unused by CLI). -- Findings: - - Trail of Bits skill routing already uses a stage named `map` (mapping-discipline skills: Semgrep/CodeQL/sharp-edge). That is not the complexity map. `stages.py` must not reuse `skills.STAGES`. - - MCP scan tools reject `deep` and forbid docker/shell. This spec does not widen the MCP surface; deep is CLI/CI only. - - `--fail-on` is implemented in `scan_exit_code` as well as argparse. `worth-fixing` must be added there. - - Java/Groovy entry points are tag-only (`line=None` → `inferred-file-surface`). Hotspot `function_name` will often be empty on Java; the map must still rank the file. - - `[sandbox]`, `[dynamic]`, `[evidence]`, and `[models].ranker` are loaded into resolved config and then ignored. Stage 3 consumes `SandboxConfig`; it does not invent a parallel sandbox TOML. -- Implications: Naming, MCP, fail-on, and Java identity are integration constraints, not new products. - -### Evidence ladder honesty -- Context: HarnessX hunter stamped LLM JSON as `static_corroboration`. Original spec forbids reporting LLM-only claims as verified. -- Sources Consulted: `verification.py` (`verify_finding` accepts when reachability is not `unknown`), `hunter_harness.py` evidence_level assignment, openrouter requirements Evidence Ladder / Verification. -- Findings: - - Pattern hits as `static_corroboration` is consistent with the ladder. LLM hits as `suspicion` is the missing rule. - - `--fail-on verified` today means reachable + static corroboration. That remains the cheap PR gate. `--fail-on worth-fixing` is the rugged gate. -- Implications: Do not change CWE policy. Change hunter evidence. Add fail-on value. Report sections: Inventory, Complexity map, Worth fixing. - -## Architecture Pattern Evaluation - -| Option | Description | Strengths | Risks / Limitations | Notes | -|--------|-------------|-----------|---------------------|-------| -| Map-then-prove on existing modes | `quick`/`standard`/`deep` = stages 1 / 1+2 / 1+2+3 | Smallest CLI blast radius; matches threat-model "deep executes in sandbox"; reuses preprocess/rank/findings | Two vocabularies (mode vs stage) until the manifest lists both | **Selected** | -| New `--stage` command family | Parallel CLI, leave modes as-is | Clearer naming | Forks CI; `deep` still a stub | Rejected | -| Clearwing-shaped full sourcehunt | Tool hunters + sanitizer images + PoC + patch | Highest detection efficiency | Fork in all but name; out of original non-goals | Rejected as this spec; remains the oracle | -| HarnessX-required agent loop | Stage 2/3 only run with the extra | Reuses last spec's seam | CI never installs it; last spec's failure | Rejected as a requirement; optional runtime only | - -## Design Decisions - -### Decision D1: Modes map to stages -- Context: Need a three-question scan without breaking CI that already passes `--mode`. -- Selected Approach: `quick`→stage 1, `standard`→1+2, `deep`→1+2+3. Manifest records both. -- Rationale: Evolutionary adoption of the CLI the project already shipped. -- Trade-offs: Operators must learn that `standard` now means "map" not "regex hunter pool." README must say so. - -### Decision D2: Docker CLI subprocess, no SDK -- Context: Zero-dep core vs isolated execution. -- Selected Approach: `docker` binary via `subprocess`; probe with `docker info`; fake runner in unit tests. -- Rationale: Operator runtime, not a Python dependency. Matches threat-model flags. -- Trade-offs: Windows/macOS Docker Desktop variance; integration tests skip without a daemon. - -### Decision D3: OpenRouter client for the tool hunter; HarnessX optional -- Context: Last spec hid the LLM behind an extra CI does not install. -- Selected Approach: Tool loop calls `OpenRouterChatClient` (or any `RankerClient`-shaped chat client) when `[models].hunter` is set. If HarnessX extra is present, `HxScanOrchestrator` may host the same tools; it is not required. -- Rationale: Evolutionary use of code already in tree. Stdlib HTTP. -- Trade-offs: Two possible runtimes until HarnessX is either used or ignored; tests mock the client. - -### Decision D4: Complexity map is a new artifact -- Context: Ranking already exists. -- Selected Approach: `ComplexityMap` with `Hotspot` records. Ranking remains the hunter budget. Map is what humans and stage 3 consume. -- Rationale: Different question (where to look / how to test vs which file to hunt first). - -### Decision D5: Worth-fixing is a gate, not an evidence level -- Context: Evidence ladder is already six rungs. Do not invent a seventh for CI. -- Selected Approach: Verdicts `triggerable|not_triggerable|already_covered|inconclusive` → `worth_fixing|not_worth_fixing`. Evidence levels unchanged. Hunter stays at `suspicion`. -- Rationale: CI owners need a fail-on that means "we proved it," without collapsing the ladder. - -### Decision D6: Split-sink / mutation gate for stages 2–3 -- Context: 90% recall on cheat sheets is circular. -- Selected Approach: New corpus + "beats pattern-hit ranking" test. Stage 1 smoke gate stays. -- Rationale: Directly answers the user's evaluation complaint. - -### Decision D7: Ledger overlays hotspot scores, not patterns -- Context: Evolutionary feedback without repeating "loop must not edit patterns." -- Selected Approach: `.openultrasast/calibration/complexity_ledger.json` keyed by `repo_fingerprint + path + function`. Triggerable boosts, not_triggerable demotes. Cannot hide reachable severity-5 inventory from worth-fixing consideration. -- Rationale: Same overlay pattern as `rule_policy.json`, different object. - -### Decision D8: Language recipes, not a universal executor -- Context: Running untrusted Python vs C vs Java is different. -- Selected Approach: Small recipe registry (Python and JavaScript required for MVP; C compile+run when a compiler exists in the image; Java `inconclusive` until a recipe ships). Missing recipe → `inconclusive`, not a crash. -- Rationale: Simplification. Do not build Clearwing's sanitizer image matrix in this spec. - -## Synthesis - -- Generalization: Inventory → attention → proof is one pipeline with a `StagePlan`. Map, hints, and regress are processors on that plan, not three products. -- Build vs adopt: Adopt Docker CLI and the in-tree OpenRouter client. Build the map, ledger, recipes, and split-sink corpus. Do not adopt Clearwing, docker SDK, or a cyclomatic-complexity library (nesting depth from indentation/braces is enough). -- Simplification: No `--stage` flag, no HarnessX requirement, no fuzz campaign, no patch oracle, no Python Docker extra. - -## Risks & Mitigations -- Docker-in-Docker / rootless CI, probe, skip stage 3, degrade; do not fail stage 1. -- Hunter cost on large repos, hotspot cap + existing `[harnessx] max_cost_usd` / hardening `max_findings` / new `max_regress_candidates`. -- Untrusted snippet execution, structural deny (network, writes outside scratch, curl, docker socket mount). Never mount `/var/run/docker.sock`. -- Stage 2 still heuristic without a model, allowed; gate is ranking quality on split-sink, which heuristics can pass if tags+reachability work. -- Breaking `deep` from "exits" to "runs", intended; document in README. - -## References -- OpenUltraSAST threat model, `docs/threat-model.md` -- OpenRouter SAST harness requirements, Docker sandbox, hunter pool, evidence ladder -- HarnessX self-improving rulesets, adjacent governance; do not reopen -- Docker run isolation flags, `--network none`, `--read-only`, `--cap-drop ALL`, `--security-opt no-new-privileges` -- Clearwing sourcehunt hunter tools and sandbox, implementation oracle, not a dependency diff --git a/.kiro/specs/three-stage-scan/spec.json b/.kiro/specs/three-stage-scan/spec.json deleted file mode 100644 index 52ef1e5..0000000 --- a/.kiro/specs/three-stage-scan/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "feature_name": "three-stage-scan", - "created_at": "2026-09-03T16:10:07Z", - "updated_at": "2026-09-03T16:47:00Z", - "language": "en", - "phase": "implementation", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true -} diff --git a/.kiro/specs/three-stage-scan/tasks.md b/.kiro/specs/three-stage-scan/tasks.md deleted file mode 100644 index 1a0352f..0000000 --- a/.kiro/specs/three-stage-scan/tasks.md +++ /dev/null @@ -1,203 +0,0 @@ -# Implementation Plan - -This plan ships the three-stage scan without rebuilding CWE policy or the regex inventory. Foundation introduces the stage contract. Core builds the map, tool hunter, sandbox, and regression verdicts in parallel where boundaries do not overlap. Integration wires them onto `quick` / `standard` / `deep`. Validation adds split-sink corpora and the anti-cheat-sheet gate. - -## Phase 1 — Foundation: Stage contract - -- [x] 1. Introduce the stage plan and mode mapping -- [x] 1.1 Encode the three stages and the mode-to-stage table - - Depth `quick` requests only static, `standard` requests static then map, `deep` requests static then map then regress. - - A skipped stage is a structured reason, not an exception, and can be serialized into the existing degradations list. - - Observable completion: a unit test asserts the three mappings and that `quick` never includes map or regress. - - _Requirements: 1.1, 1.2, 1.3, 1.6_ - - _Boundary: StagePlan_ - -- [x] 1.2 Add complexity and regression configuration with candidate caps - - Configuration exposes map top-k, max hunter hotspots, max regression candidates, and optional language image pins, with defaults that keep pull-request scans cheap. - - Sandbox memory, pids, timeout, network-off, and read-only workspace remain the existing sandbox settings. - - Observable completion: loading config without a file yields those defaults; an override file changes top-k and max candidates only. - - _Requirements: 9.2, 9.5_ - - _Boundary: Packaging Configuration_ - - _Depends: 1.1_ - -- [x] 1.3 Expand fail-on vocabulary with worth-fixing - - The scan command accepts `worth-fixing` alongside the existing fail-on values and does not change the meaning of `findings` or `verified`. - - The existing report exit helper must recognize the new value; unknown values still fail loud. - - MCP continues to reject `deep` and must not gain docker or shell tools. - - Observable completion: `--help` lists `worth-fixing`; a scan with no worth-fixing verdicts and `--fail-on worth-fixing` exits 0; MCP still refuses `deep`. - - _Requirements: 5.7, 5.8_ - - _Boundary: StagePlan_ - - _Depends: 1.1_ - -## Phase 2 — Core: Map, hunter tools, sandbox (parallel after foundation) - -- [x] 2. Build the heuristic complexity map and test-tuning hints -- [x] 2.1 (P) Compute per-file and per-function complexity signals - - Signals include size, nesting, surface tags, reachability, inventory-hit density as a feature not the score, and whether an adjacent test appears to exist. - - Function identity uses entry-point names when present and otherwise remains empty. - - Observable completion: a parser file with no pattern hit scores higher than a one-line helper that matches a pattern, proven by a unit test on a constructed target list. - - _Requirements: 3.1, 3.2_ - - _Boundary: ComplexityMapBuilder_ - -- [x] 2.2 (P) Emit the ordered hotspot map artifact - - Each hotspot carries score, band, rationale, signals, and linked inventory ids; the map records whether it is heuristic-only. - - Observable completion: building a map writes a JSON artifact whose hotspot order is not identical to sorting by inventory-hit count on the same fixture. - - _Requirements: 3.2, 3.3, 3.5, 8.6_ - - _Boundary: ComplexityMapBuilder_ - - _Depends: 2.1_ - -- [x] 2.3 Attach test-gap hints without writing into the scanned tree - - Each hotspot gets a gap of no adjacent test, no function reference, no fuzz entry, or covered. - - Recommended test kind is one of unit, property, sanitizer, http-contract, or fuzz-harness, and is omitted when the gap is covered. - - Observable completion: a covered hotspot emits `covered` and no test kind; a fuzzable C parser without a harness emits `fuzz-harness`; the scanned tree is unchanged. - - _Requirements: 4.1, 4.2, 4.3, 4.4_ - - _Boundary: TestTuningAdvisor_ - - _Depends: 2.2_ - -- [x] 2.4 Apply a complexity ledger overlay that cannot hide severity-5 reachable inventory - - Triggerable verdicts raise a hotspot's later score; not-triggerable verdicts lower it; the underlying inventory hit remains. - - A reachable policy-severity-5 inventory finding stays in the regression candidate set regardless of a negative ledger delta. - - Observable completion: after a not-triggerable overlay the hotspot score drops, and a sev-5 reachable finding is still selected as a candidate, proven by a unit test. - - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5_ - - _Boundary: ComplexityMapBuilder_ - - _Depends: 2.2_ - -- [x] 3. Add repo-bound hunter tools and a scripted tool loop -- [x] 3.1 (P) Clamp tool paths and expose read, grep, and reference search - - Read, grep, and reference listing resolve only under the scan root; `..` and absolute escapes raise a path-escape error. - - Grep runs with the standard library over enumerated source files. - - Observable completion: a unit test reads a repo file, greps a split-sink assignment, and rejects `../etc/passwd`. - - _Requirements: 6.1, 6.2_ - - _Boundary: HunterTools_ - -- [x] 3.2 Run a bounded tool-using hunter that records suspicion only - - When a hunter model is configured, the loop may call the three tools against map hotspots and must not stamp results as static corroboration. - - When no model is configured, the loop does not run and a degradation is produced by the orchestrator later. - - Observable completion: a scripted client that never calls a tool fails the test; a client that greps then returns a finding yields evidence level `suspicion` and id prefix `tool-hunter:`. - - _Requirements: 6.1, 6.3, 6.4, 6.5_ - - _Boundary: ToolHunter_ - - _Depends: 3.1, 2.2_ - -- [x] 4. Stand up the isolated sandbox runner -- [x] 4.1 (P) Probe whether the sandbox runtime is usable - - A short `docker info` style probe returns available or unavailable without throwing into the scan. - - Observable completion: the fake probe can be forced off; the real probe is skippable in unit tests. - - _Requirements: 9.3_ - - _Boundary: SandboxProbe_ - -- [x] 4.2 (P) Run jobs through Docker CLI flags that match the threat model - - Network is disabled, source is read-only, scratch is writable, memory/pids/timeout apply, capabilities are dropped, source tree is not writable, and the host Docker socket is never mounted. - - A fake runner records the job and returns a programmed result so default tests do not need a daemon. - - Observable completion: the constructed command line includes network-none and read-only and excludes docker.sock; the fake runner is used by unit tests. - - _Requirements: 5.2, 5.8_ - - _Boundary: DockerCliRunner_ - - _Depends: 4.1_ - -- [x] 4.3 Reject unsafe regression snippets before execution - - Snippets that refer to the Docker socket, host network, or writes under the source mount are not executed and become `inconclusive`. - - Observable completion: each forbidden shape is rejected in a unit test and no runner job is created. - - _Requirements: 5.8, 6.6_ - - _Boundary: RegressionRunner_ - - _Depends: 4.2_ - -## Phase 3 — Core: Regression verdicts - -- [x] 5. Produce worth-fixing verdicts from sandboxed candidates -- [x] 5.1 Select a bounded candidate set from hotspots and severe reachable inventory - - Selection honors max-candidates and always includes reachable severity-5 inventory even when the ledger demoted the hotspot. - - Observable completion: a unit test with six hotspots and max-candidates three returns three, plus the sev-5 reachable extra when present. - - _Requirements: 5.1, 7.4, 9.5_ - - _Boundary: RegressionRunner_ - - _Depends: 2.4_ - -- [x] 5.2 (P) Add language recipes with inconclusive fallback - - Python and JavaScript recipes run a scratch snippet with the repo on the language path; C compiles and runs when the image has a compiler; a missing recipe yields `inconclusive` rather than a crash. - - Observable completion: Python and JavaScript recipes produce a job; an unknown language returns inconclusive without calling the runner. - - _Requirements: 5.1, 5.6_ - - _Boundary: RegressionRunner_ - - _Depends: 5.1_ - -- [x] 5.3 Map sandbox outcomes to the four verdicts and the worth-fixing gate - - Outcomes map to triggerable, not_triggerable, already_covered, or inconclusive; worth-fixing requires triggerable plus reachable or inferred-file-surface. - - Not-triggerable and already-covered do not fail the build; inconclusive records a reason and does not fail the build. - - Observable completion: a table-driven test covers the four verdicts and the worth-fixing boolean. - - _Requirements: 5.3, 5.4, 5.5, 5.6_ - - _Boundary: WorthFixingGate_ - - _Depends: 5.2, 4.2_ - -## Phase 4 — Integration: Wire stages into the scan - -- [x] 6. Run stages from the existing scan command -- [x] 6.1 Keep quick as stage-1 inventory with no model and no target execution - - Quick still loads policy, ruleset, preprocess, rank, pattern inventory, structural verify, and score. - - The report labels those hits as inventory, not as worth-fixing. - - Policy still fails loud on an unmapped enabled CWE. - - Observable completion: existing quick tests stay green; the report contains an inventory section and does not start a sandbox or a hunter model. - - _Requirements: 1.1, 2.1, 2.2, 2.3, 2.4, 2.5, 9.1_ - - _Boundary: StagePlan_ - - _Depends: 1.1_ - -- [x] 6.2 Run the complexity map on standard and record heuristic-only when no model is set - - Standard writes the map artifact, includes it in the manifest, and records `hunter_model_unavailable` when the hunter is skipped. - - Observable completion: `ousast scan --mode standard` on a fixture produces `complexity_map.json` and a completed map stage without Docker. - - _Requirements: 1.2, 1.4, 3.3, 3.4, 6.4, 9.4, 9.6_ - - _Depends: 2.3, 6.1_ - -- [x] 6.3 Stop exiting on deep; run or skip regression with visible degradation - - Deep no longer hard-exits. If the sandbox is missing, stages 1–2 complete and regress is skipped with `sandbox_unavailable`. - - If the sandbox is present, regression runs under the candidate cap and writes verdicts. - - Observable completion: `ousast scan --mode deep` without Docker exits 0 after stages 1–2 and lists the skip in the manifest; with a fake runner it records verdicts. - - _Requirements: 1.3, 1.5, 5.1, 9.3, 9.6_ - - _Depends: 5.3, 6.2_ - -- [x] 6.4 Attach the tool hunter to map hotspots when a model is configured - - Hunter findings are suspicion-only and may propose a snippet that still must pass the safety check and run only in the sandbox. - - Observable completion: with a scripted client, standard+model produces tool-hunter suspicions; a proposed snippet is executed only on deep after the safety check. - - _Requirements: 6.1, 6.3, 6.5, 6.6_ - - _Depends: 3.2, 6.2_ - -- [x] 6.5 Surface map, hints, and worth-fixing in the report and honor fail-on - - Markdown, manifest, and SARIF remain; the report gains map and worth-fixing sections. - - `--fail-on worth-fixing` fails only when a worth-fixing verdict exists. - - Observable completion: a scan with one worth-fixing verdict fails under that flag and passes under `--fail-on never`. - - _Requirements: 2.2, 4.1, 5.7, 7.3_ - - _Depends: 6.3_ - -## Phase 5 — Validation: Grounded corpora and CI - -- [x] 7. Replace cheat-sheet success for stages 2 and 3 -- [x] 7.1 Add split-sink fixtures for Python, JavaScript, Java, and C/C++ - - Each language family has at least one case where source and sink are on different lines or assigned before use. - - Observable completion: stage-1 pattern scan misses at least one planted sink in each family, documented in the manifest expected set. - - _Requirements: 8.1_ - - _Boundary: Benchmark Harness_ - -- [x] 7.2 (P) Add a mutation of a known sink that a same-line pattern does not match - - At least one wrapper or intermediate-variable mutation exists per the evaluation plan. - - Observable completion: stage 1 misses the mutation; the case is listed as expected in a split-sink or mutation manifest. - - _Requirements: 8.2_ - - _Boundary: Benchmark Harness_ - -- [x] 7.3 Gate stage 2 on ranking the true function in the high band and beating pattern-hit order - - Stage 1 may still use existing smoke fixtures; that recall is not treated as stage-2 success. - - The merge gate fails if map order equals pattern-hit-count order on the split-sink corpus. - - Observable completion: a dedicated gate command or test fails a stub map that sorts by hit count and passes the heuristic map on the split-sink corpus. - - _Requirements: 8.3, 8.4, 8.6_ - - _Depends: 7.1, 2.2_ - -- [x] 7.4 Prove stage 3 can trigger a planted miss when the sandbox is available - - Docker-marked tests skip when the probe fails; when they run, at least one stage-1 miss is `triggerable`. - - Default pytest does not require a daemon. The pull-request job stays stage 1; a nightly or labeled job may request deep under time and candidate caps. - - Observable completion: fake-runner tests always run; docker tests skip-or-pass; the default CI workflow still invokes quick-only detection plus the new split-sink map gate. - - _Requirements: 8.5, 9.1, 9.2_ - - _Depends: 6.3, 7.1_ - -## Implementation Notes - -- `RegressConfig.images` is a sorted tuple of `(language, image)` pairs so frozen config stays hashable. -- `scan_exit_code` duck-types `worth_fixing` on verdict objects; `_run_scan` still passes no verdicts until task 6.5. Dict-shaped verdicts would not fail the gate. -- Map `top_k` default is 20 (not named in design.md); `max_hunter_hotspots=8` and `max_candidates=5` follow design. -- `collect_signals` adjacent-test detection is false unless the caller also passes `repo_files`; task 2.2 must pass enumerated repo paths. -- ToolHunter rejects JSON dumps unless a repo tool was invoked; findings are always `suspicion` with prefix `tool-hunter:`. -- Split-sink map gate treats "high band" as the highest band the map emitted on that corpus (small fixtures often top out at medium). -- `@pytest.mark.docker` tests skip when `SandboxProbe` is unavailable; default CI does not require a daemon. diff --git a/.kiro/specs/unseen-repo-evaluation/brief.md b/.kiro/specs/unseen-repo-evaluation/brief.md deleted file mode 100644 index 0d8dcf9..0000000 --- a/.kiro/specs/unseen-repo-evaluation/brief.md +++ /dev/null @@ -1,32 +0,0 @@ -# Brief: does the safety net work on repositories it has never seen? - -Maintainer, 2026-10-04: "we want to spread the safety net across repos in general, unseen code. that was the purpose -of the AB tests to see if our approach is generalizeable." And: "we should run our tests / runs in kube. now we can -speed it up and design the workloads for ax directly. kind is only a fallback." - -## Why this spec exists - -The A/B experiments so far (exp-003, exp-004, exp-005) score the decision engine on repository-held-out folds of known -fix pairs. That measures whether the vulnerable side of a pair scores above its fixed side. It does not measure what a -safety net on an arbitrary repository lives on: - -- on an ordinary push, does it stay quiet? (false-alarm rate) -- on a push that introduces a real vulnerability, does it say so? (catch rate) - -Populations v1 and v2 measured the engine scan on fresh cases once (1 of 11, 0 of 17) and are spent. Nothing since has -measured push-level behaviour on unseen repositories. - -## What this spec delivers - -1. A frozen pool of repositories never used anywhere in development, with two kinds of change each: real - vulnerability-introducing commits and ordinary commits. -2. A push-level metric: each change replayed through the hook exactly as a user would run it, and through other - arms (decision engine, engine evidence) where an experiment asks. -3. Execution designed for the kube-ax cluster from the start: one push replay is one AX Task; kind and the VM are - fallbacks. -4. A baseline for today's hook, then a table of arms scored on the same pool. - -## Not in scope - -Changing detection. This spec only builds the measurement. Improvements are separate, registered experiments judged -by it. diff --git a/.kiro/specs/unseen-repo-evaluation/design.md b/.kiro/specs/unseen-repo-evaluation/design.md deleted file mode 100644 index ae8a20f..0000000 --- a/.kiro/specs/unseen-repo-evaluation/design.md +++ /dev/null @@ -1,518 +0,0 @@ -# Design Document - -## Overview - -This spec builds the instrument for roadmap G0: a frozen pool of repositories that no development step has seen, -and a push-level score on it. The score is what a user of the hook sees. On a vulnerability-introducing push, did -the hook say so (catch rate)? On an ordinary push, did it say anything (false-alarm rate)? Replays run as AX Tasks -on kube-ax. kind and the VM run the same tasks as fallbacks. - -Four facts from the code shape the design: - -- **Populations already have a freeze-and-guard pattern.** `benchmarks/independent/freeze.py` verifies cases by - reading git only, `evaluate.py` scores with fixed rules and Wilson intervals, and - `tests/test_independent_population.py` fails when a reserved repository is named anywhere in `benchmarks/`, - `src/` or `tests/` outside its directory. Harvest batch 3 used that guard as a black box: candidates written, - guard run with output discarded, subsets bisected on red. That is how v3 stays unread. -- **The hook can already replay a change.** `ousast pre-push PATH --base B --head H --artifact F` resolves two - local commits (`snapshot.py`, `resolve_replay`: no fetch, no merge-base), runs quick rules and the engine inside - one deadline (default 30 s, engine inline) and writes a JSON artifact. The terminal text has one `Skipped:` line - per skipped check. The artifact holds three finding streams: admitted defects, advisory engine findings - (new or worsened, vetoed only by capability) and quick-rule matches with their CWE. -- **The AX pattern works but lives in one script.** `engine_trace_ax.py` (manifest, egress checks, presigned - objects, retry once under a new name, cleanup) and the lane scheduler in `engine_trace.py` (`run_plan`: fixed - lanes, fallback queue, `progress.json`, `STOP`, resume by input digest) are engine-specific. -- **Placement on kube-ax is random.** One worker per node, shared by all actors. Two heavy tasks fit per worker, - but only two concurrent tasks in all guarantee at most two on one worker. - -### Goals - -- A pool frozen by code against every used source (Req 1), with two kinds of change per repository (Req 2). -- A scorer whose numbers mean what the hook shows, with silent zeros visible (Req 3). -- One replay is one AX Task via a dispatcher shared with the engine (Req 4); a baseline and fresh slices (Req 5). - -### Non-Goals - -Detection changes, population v3, model calls in the cluster, languages the hook does not analyse. - -## Architecture - -```mermaid -flowchart LR - M[pool manifest slice] --> D[batch dispatcher on VM] - D --> P[presign spec GET and record PUT] - P --> T[AX Task ousast-engine-replay] - T --> G[shallow fetch base and head from GitHub] - G --> H[ousast pre-push --base --head] - H --> R[record tar: artifact, text, instrument] - R --> S[(store via presigned PUT)] - D -->|sandbox died twice or OOM| V[VM docker lane] - D -->|cluster down| K[kind lane, same tasks] - V --> S - K --> S - S --> C[scorer: catch, false alarm, coverage] - S --> A[model arms on VM, budgeted] - A --> C - C --> O[record: counts and digests] -``` - -## Components and Interfaces - -### 1. Pool sourcing and eligibility (Req 1) - -**Decision 1.1: sources and ecosystems.** Candidates come from GitHub reviewed advisories (`gh api /advisories`, -the enumeration harvest-3 used) for pip, npm, maven and composer, published 2019 or later, whose CWE maps to one of -the seven families through the project's family CWE table. Each needs a linked fix commit with one parent in one -repository. OSV records for the same advisory give the first affected version. That version is a cross-check for -the introducing commit (section 2). Reason: these four ecosystems are the five languages the hook analyses -(Python, JavaScript, TypeScript, PHP, Java). GHSA links commits and CWEs; OSV adds version brackets. Rejected: -drawing from every ecosystem (Go, Ruby, C#). That would mostly measure missing frontends, which is already known. -Coverage still reports uncovered files inside the drawn repositories. Rejected: NVD and published fix datasets. -NVD links commits weakly, and the datasets are mined corpora that development or model training has likely seen. - -**Decision 1.2: eligibility by code against every used source.** `benchmarks/unseen/eligibility.py` builds the -used set before the draw. It never prints names; it records counts per source and a digest of the set. - -| Source (Req 1.1) | Where it is read | -| --- | --- | -| Pair corpora | every catalog, recipe and dataset file under `benchmarks/pairs/`, plus the local pointer-pair manifests | -| Populations v1, v2 | `population-v1.toml`, `population-v2.toml`, read directly | -| Population v3 | never opened; the existing guard test only (below) | -| Harvest records | the batch catalogs under `benchmarks/pairs/advisory-fixes*` (the records hold counts only) | -| Memory examples | `example` rows of the local FileStore and the S3 store (`repo` field) | -| Experiment units | `plane/experiments/*.units.jsonl` and registered units files | -| Measurement records | everything under `benchmarks/measurements/` and `benchmarks/experiments/` | -| Anything else | a text sweep for `github.com//` over `benchmarks/`, `src/`, `tests/`, `plane/`, `.kiro/` | - -Names are normalised (lower case, no `.git`, no trailing slash). Each candidate is resolved through the GitHub API -to its canonical `full_name`, its former names (redirects) and its fork network (`parent`, `source`). A candidate -is out if any of these is in the used set. A fork shares code with what it was forked from. - -v3 is checked like harvest-3 did. The draft manifest is written into `benchmarks/unseen/`. Then -`pytest -q tests/test_independent_population.py -k referenced` runs with its output discarded. The guard already -sweeps `benchmarks/` outside the population directory. On red, the candidate list is bisected by rewriting the -draft until green, and the dropped count is recorded. No v3 name is ever printed or read by a person. - -**Decision 1.3: post-cutoff preference.** An advisory published after 2026-06-01 (harvest-3's cutoff for the -decision engine's model) is flagged `post_cutoff`. Within each ecosystem stratum post-cutoff advisories are drawn -first. Results are reported for both groups. Reason: a post-cutoff advisory cannot be in the model's training -data as a known vulnerability. Rejected: post-cutoff only. Harvest-3 kept 12 post-cutoff pairs out of 251, and -three slices of 100 repositories cannot be filled from that window. - -**Decision 1.4: licences.** The licence is read at the pinned head commit (LICENSE file, then the API's SPDX id). -Only permissive licences are published by name: MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD, Unlicense, -Zlib, PostgreSQL, BSL-1.0. Everything else (copyleft, source-available, none) goes to the gitignored -`benchmarks/unseen/private/pool-p1.toml`. The committed manifest then holds an opaque id and the sha256 of -(URL, commits) for that entry, so the freeze digest still covers it. Reason: Req 1.4 read literally. Rejected: -"OSI-approved" as the bar, as population v2 used. It is wider than the requirement says (open question 1, -resolved 2026-10-04: copyleft stays private). - -Also required: anonymous HTTPS, at most 500 MB, at least 40 eligible ordinary commits, one per fork network. - -### 2. Change extraction (Req 2.1, 2.2) - -Extraction runs on the VM over a full `--filter=blob:none` clone in a scratch cache. It uses git only, like -`freeze.py`. No scanner, engine or model touches a pool repository before the freeze. - -**Decision 2.1: the vulnerability-introducing range.** Let F be the fix and P its parent. The vulnerable lines are -the old-side lines of F's hunks in the vulnerable function. For a pure-addition fix (an absence bug such as a -missing guard), they are the vulnerable function's own lines. - -1. **`introducing` (preferred).** `git blame -w -M -C --porcelain P -L -- `. The candidate V is the - commit that last wrote the sink line (for absence bugs, the function's declaration). V is accepted if all hold: - at V's first parent the sink line's normalised text is absent from the file's vulnerable function; V is not a - merge or root commit; V changes at most 50 files and 2,000 lines; when OSV names a first affected version, V is - reachable from that tag and not from the previous release tag. If the text is present at V's parent, blame - walks back from there, at most five steps. The replay is `base = V^`, `head = V`. -2. **`last_touch` (fallback).** When no V passes, `git log -L :: P` gives the newest commit L - before F that changed the vulnerable function. The replay is `base = L^`, `head = P`. The vulnerable code is in - head, and the range contains a change to the vulnerable function. - -Each change records its method, family, vulnerable function (resolved in head), head-side changed lines, -`post_cutoff` and the advisory id. - -Bias of each, stated in the record. `introducing` inherits SZZ noise: blame finds the last writer, not the author -of the flaw. A refactor, a move the heuristics miss or a reformat can stand in for the real introduction. The -absence check, the walk-back and the version bracket cut this down, but do not remove it. The bias runs both -ways: a wrong V may introduce nothing (a miss is not the hook's fault) or may move a flaw (the delta-based hook is -right to call it existing). `last_touch` is pessimistic for a delta hook. The flaw is often already in base, so -the hook's novelty rule correctly stays quiet and the change counts as a miss. Catch is therefore reported per -method. The headline is `introducing`; `last_touch` is a lower bound. Rejected: fix-parent against a fixed old -base (for example 50 commits back). The range often misses the vulnerable function entirely and grows without -bound on busy repositories. - -Label check: every third `introducing` change is read by an agent before the freeze, with no model call, as -harvest-3 reviewed diffs. It is marked `confirmed`, `refactor` or `unclear`. The confirmed share is the label -noise estimate. The scorer also reports catch on the confirmed subset. - -**Decision 2.2: ordinary commits.** The candidates are non-merge commits on the first-parent line of the default -branch that change at least one source file of an analysed language. Excluded: - -- explicit security signals in the message: `secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|exploit|sanitiz|escap`; -- reverts (`^Revert`, `This reverts commit`); -- dependency bumps (bot authors such as dependabot and renovate, `bump|upgrade .* from`, or only manifest and lock - files changed); -- commits whose source changes are all generated or vendored (`vendor/`, `node_modules/`, `dist/`, `*.min.js`, - lock files, `linguist-generated` in `.gitattributes`, "generated" headers); -- any fix commit of any advisory of the repository, and any commit touching a known vulnerable function of it; -- commits over 1,000 changed source lines (counted, reported). - -Twelve are drawn per repository with a seeded random draw. Strata are three time thirds of the repository's -history crossed with four size buckets of changed source lines (1-10, 11-50, 51-200, 201-1,000). Allocation is -proportional to the repository's own commits per cell, with at least one per time third. Each change records base -(its first parent), head, changed files and line ranges. Reason: proportional allocation keeps the false-alarm -rate a per-push rate as a user meets it. Rejected: `freeze.py`'s broad `SECURITY_WORDS`. It also drops commits -saying "query", "token", "session" or "valid", which removes the ordinary commits where false alarms are most -likely, and the rate would look better than it is. Rejected: docs-only commits. They are trivially quiet and -would dilute the rate. - -Label noise: an ordinary commit may itself introduce an unknown vulnerability, so a flag counted as a false alarm -may be true. The false-alarm rate stays as Req 3.2 defines it. Flagged ordinary changes are adjudicated beside it -(all if 30 or fewer, else every third) against protocol-v2's standard, and the adjudicated share is reported. - -### 3. Sizing and slices (Req 2.3, 5.2) - -Wilson 95% interval, z = 1.96: centre (p + z²/2n)/(1 + z²/n), half width z·sqrt(p(1-p)/n + z²/4n²)/(1 + z²/n). -Taking the larger distance from p as the width, the smallest n that meets the bar, computed exactly: - -| Rate | Assumed p | Bar | n needed | -| --- | --- | --- | --- | -| false alarm | 0.05 | +/- 3 points | 315 | -| false alarm | 0.10 | +/- 3 points | 483 | -| false alarm | 0.20 | +/- 3 points | 756 | -| catch | 0.10 | +/- 10 points | 62 | -| catch | 0.25 | +/- 10 points | 88 | -| catch | 0.40 | +/- 10 points | 97 | - -Catch: 100 vulnerability-introducing changes per slice, one per repository. At p = 0.4, Wilson gives -[0.309, 0.498], within 10 points. One per repository keeps them independent. - -False alarm: ordinary changes of one repository are correlated (noisy code flags often). Wilson assumes -independence, so the count is inflated by a design effect 1 + (m - 1)·ICC. With m = 12 per repository and an -assumed ICC of 0.05, that is 1.55. 100 repositories × 12 = 1,200 changes, effective n 774 ≥ 756. The plain Wilson -interval at p = 0.2, n = 1,200 is about +/- 2.3 points. A repository-cluster bootstrap interval is reported beside -it, and the ICC measured on slice 1 is recorded. If it exceeds 0.05, the next pool sizes up. A frozen slice is -never changed. - -**Decision 3.1: slices.** Pool p1 has three slices of 100 repositories each (100 + 1,200 changes per slice; 300 -repositories and 3,900 changes in all). Repositories are assigned by a seeded shuffle, stratified by ecosystem and -`post_cutoff`. Slice 1 serves the baseline and the G1 arms table. Slices 2 and 3 serve the first two adoption -decisions. When they are spent, pool p2 is drawn by the same code, and it excludes p1. Reason: each slice alone -meets the Req 2.3 widths, so any decision can stand on one slice. Rejected: one large pool scored whole. Every -decision would then see every repository, which Req 5.2 forbids. Rejected: ten small slices. None would meet the -widths. Per family, 100 changes split over seven families give intervals of about +/- 20 to 25 points. They are -reported, not sized (open question 3). - -Use ledger: `benchmarks/unseen/usage-p1.jsonl` is append-only, one row per use (slice, decision or experiment id, -date, result digest). A slice with a row whose purpose is `informed` cannot appear again as `qualifies` for that -change. A test checks this. - -### 4. Scoring rules (Req 3.1, 3.2, 3.3) - -Each replay uses the hook's defaults: `--deadline 30`, `--engine inline`, `--max-regions 500`, `--mode advisory`. -That is what a user's push gets. The text output is kept with every `Skipped:` line, and so is the artifact. - -**Decision 4.0: a registered long-deadline arm** (maintainer decision 2026-10-04, open question 5). Beside the -default replay, slice 1 is replayed once more with `--deadline 300` and every other flag unchanged. The arm is -registered (manifest committed, `unseen: {pool: p1, slice: 1}`) before its first replay. G1 then reports both -what the user sees (30 s) and what the engine could catch given time (300 s), so a miss caused by the deadline is -told apart from a miss by the engine. It is a measurement arm, not a proposed default: adopting a longer hook -deadline would be a separate decision under 7.2. - -**Decision 4.0a: calibrated cluster deadline and native check** (maintainer, 2026-10-05). -The gVisor proof exhausted 30 seconds in snapshot preparation on a 1,725-file repository. -Cluster replays therefore scale the 30/300-second arms by the median paired cluster/native -time through the quick tier, rounded to integer seconds. Native runc replays retain 30/300 seconds. -A deterministic sample (default 20, or all of a smaller proof manifest) runs both lanes at -600 seconds for calibration. Report per-change stage times and median slowdown with bootstrap -95% intervals for time through quick, engine build and engine queries separately. Incomplete -timings or degraded scans invalidate calibration. Records contain counts, times and opaque digests. - -A deterministic native sample accompanies the cluster slice. Compare whether each push flags -anything, with a Wilson 95% interval. If its lower bound is below 95%, or any sample pair is -missing/failed, flag the run and exclude cluster results from the baseline. Disagreements name -change IDs only. The scorer reports applied deadlines and factors. Docker pulls explicitly after -checking its data-root free space against an unpacked image size upper bound plus 1 GiB; failures -retain sanitized stderr tails and exit codes. See the unseen README for coordinator commands. - -**Decision 4.1: catch, per family.** A finding counts if its family is the change's family. Engine streams carry a -family; a quick-rule match maps its CWE through the family CWE table. Its head location must also be in the -vulnerable function or within 10 lines of the change's head-side hunks (protocol v1's window). - -| Family | Location rule | -| --- | --- | -| injection, path, deserialization, output_encoding, untrusted_destination | sink line in the vulnerable function, or the changed lines +/- 10 | -| access_control | the vulnerable function itself (the handler that lacks the check), or the changed lines +/- 10 | -| config_secrets | the changed file and lines +/- 10 (file scope; function ``) | - -All three streams count: admitted defects, advisory engine findings and quick-rule matches. Each catch also names -the stream that made it. Reason: the hook shows all three to the user. Rejected: admitted defects only. That -scores the blocking path, which G3 decides separately, and would hide what users read. - -**Decision 4.2: false alarm.** An ordinary change is a false alarm when any of the three streams holds at least one -finding, of any family, anywhere. The alerts-only rate (admitted defects) is reported beside it. - -**Decision 4.3: coverage, so zeros cannot hide.** Every change gets exactly one state: - -- `instrument_failure`: no record, a failed fetch, the head commit not verified, or zero bytes of the changed files - read. It is re-run once, then reported. It never counts as quiet or missed. -- `unanalysable`: no changed file is in a language the hook covers (the hook's `language_not_covered` notice). -- `quick_only`: the quick tier read the changed files, but no engine question completed on them. -- `engine_covered`: at least one engine question completed on a changed file. - -The task entry proves its input as AGENTS.md requires. It records files and bytes in the checkout, bytes of each -changed file, the JVM's wall time and the hook's exit code. A replay faster than a JVM can start is flagged -`suspect_fast` and goes to the failure path. Catch and false alarm are reported over all changes and over each -coverage state. - -### 5. The AX workload (Req 4) - -**Decision 5.1: the image.** `plane/Dockerfile.engine-task` gains a second stage, `FROM engine AS replay`, built -with `--target replay` and pinned by digest. It adds `git`, the package with its runtime dependencies and -`benchmarks/unseen/replay_entry.py`. Joern, php-cli, Java 21 and the frontend-retention patch come from the engine -stage, so both lanes run the same Joern. The import smoke runs `ousast pre-push --help`. The root image stays -`ghcr.io/norandom/ax-task-runner:v0.3.1`. Rejected: a separate Dockerfile. It would duplicate the Joern install, -and two Joern copies are two instruments that drift. Rejected: one stage for both. The engine image's isolated -venv without yaml and boto3 is proven, and replay dependencies would change its digest on every bump. - -**Decision 5.2: the task entry.** `replay_entry.py` is standard library only, like `engine_task_entry.py`, and -reuses its `transfer` (egress readiness wait, three attempts, no URL text in errors). - -1. GET `SPEC_URL` (presigned, `.json`): repository URL, base, head, change id, hook flags. -2. `git init`, then `git fetch --depth 1 origin ` and `git fetch --depth 1 origin ` over HTTPS - (GitHub serves commits by SHA). Verify both with `git cat-file -e`. The fetch is bounded at 180 s; the 500 MB - repository cap bounds its size. -3. Run `ousast pre-push --base --head --artifact /tmp/out/push.json` with defaults, the - backend heap at 1700 MB and the cluster JVM flags of `engine_trace_worker.py`. Capture stdout. -4. Pack `push.json`, `push.txt`, `engine.json` (the head and base scan records and witness rows from the artifact, - keyed by repository and revision, for engine-evidence arms) and `instrument.json` (section 4.3, image digest, - step timings). PUT to `RESULT_URL` as `.dat`. The PUT is completion. - -On any error it PUTs a failure record naming the step and HTTP status, as the engine entry does. Task names are -`ousast-engine-replay--`, the prefix the operator's egress watcher matches. No key or store -credential enters the manifest. Only the two presigned URLs do, and the spec object keeps repository names out of -AX status and cluster telemetry. - -**Decision 5.3: the generic dispatcher.** `benchmarks/ax/batch.py` takes over from `engine_trace_ax.AX` and -`engine_trace.run_plan`: - -- `Item`: id, input objects (name to bytes), the result object name, command, extra env, input digest. -- `Workload`: image digest, allowed URL env names, deadline, how to validate a received result. -- Lanes: `ax` (manifest, apply, poll the result object and AX phase, delete, clean objects), `kind` (the same - manifest against the kind AX), `docker` (the VM image, one container at a time). -- Scheduler: two AX lanes plus one VM lane by default. A sandbox death is retried once under a new name; a second - death or a JVM OOM goes to the VM fallback queue, which beats overflow on the VM lane. `progress.json` is written - only by the scheduler. A `STOP` file halts new launches. Resume skips items whose stored record has the same - input digest. Cleanup failure stops dispatching. - -The engine pass becomes a workload over it, with the same tests. Reason: the egress, secrecy and cleanup logic took -several fixes and must exist once. Rejected: a replay-specific copy of `engine_trace_ax.py`. Rejected: putting it in -`src/` now. G4's `ousast plane scan` may need it there, and it moves when G4 is planned. - -Concurrency is two AX tasks. Placement is random, and only two guarantees two per worker at most. Four would need -the operator to add placement (open question 2, resolved 2026-10-04: stays at two, no placement change -requested). Estimated wall time per slice: about 75 s per replay (fetch, 30 s hook, setup), 1,300 replays over -three lanes, about 9 hours. The long-deadline arm (4.0) is bounded at about 345 s per replay; at that bound 1,300 -replays over three lanes take about 42 hours, less where the hook finishes early. It runs with `STOP` and resume -across sessions. - -**Decision 5.4: model arms on the VM.** Arms that need DeepSeek (the decision engine, the combiner) read the stored -records on the VM. They build candidates from the head side's changed functions, score them, and take the change -score as the maximum. Each runs under its experiment manifest's `budget_usd` through the metered client, dry run -first, with responses cached. Nothing in the cluster calls a model. - -**kind (Req 4.5)** runs the identical manifest and image through the `kind` lane. Every new task is proven there -before kube-ax, as `plane-on-kubernetes` requires. - -### 6. Records and the guard (Req 1.2, 1.3, 5.1) - -`benchmarks/unseen/` holds `pool-p1.toml` (repositories, licences, slices, changes with base, head, method, -family, function, lines), `freeze-p1.json` (used-set digests and counts per source, guard result, label-check -counts, freeze digest = sha256 of the canonical manifest), `protocol-p1.md` (the scoring rules of section 4, -committed before the first replay) and `usage-p1.jsonl`. The `private/` directory is gitignored. - -The guard grows by one parametrised reservation. The existing sweep becomes a list of (reservation directory, -names): `benchmarks/independent/` for populations and `benchmarks/unseen/` for pools, including the private -manifest when present. It also sweeps `plane/`. A second test asserts that pool and population names are disjoint, -and prints only a count on failure. A third asserts that a frozen pool's digest matches `freeze-p1.json`, so the -pool is never edited. Raw per-change records stay outside the repository (`~/ousast-results/unseen/` and the -store). Committed result records under `benchmarks/measurements/` carry counts, rates, intervals and digests, -never names. - -### 7. Metrics and reporting (Req 3.3, 3.4, 5.3) - -`benchmarks/unseen/score.py` reports per family and overall: catch and false-alarm rate with Wilson 95% -intervals, the cluster bootstrap interval for false alarms, coverage counts per state, catch per method, -post-cutoff and per stream, and the alerts-only rate. - -**Decision 7.1: arms at fixed budgets.** An arm with a score is thresholded per budget (1%, 5%, 10%). The -threshold is the lowest score whose false-alarm rate on the slice's ordinary changes stays within the budget. -Catch at that threshold is reported with its Wilson interval and the realised false-alarm rate. That in-slice -threshold is labelled exploratory. For a qualification, the threshold is fixed beforehand on an earlier slice or -on held-out folds, and it is applied unchanged. A binary arm (today's hook) has one operating point, reported with -the budgets it meets. Rejected: AUC over changes (it averages operating points no user runs). - -**Decision 7.2: next to held-out folds.** Experiment manifests registered after this spec lands carry -`unseen: {pool, slice}`. `analyse` writes an `unseen` block into `result.json` next to the fold estimates: catch at -each budget for each arm, and the paired difference B - A with an exact McNemar test on the 100 changes. The -fold rule decides as today. The slice is recorded in the usage ledger. - -**Decision 7.3: adoption gate on the unseen slice** (amended 2026-10-04, maintainer decision; Req 5.4; open -question 4 resolved). Reporting is not enough. An arm B is adopted over the production arm A only if, on the -untouched slice that qualifies it, both hold: - -- catch at the 5% budget, with B's threshold fixed beforehand (7.1), is not significantly lower than A's: exact - two-sided McNemar on the paired vulnerability-introducing changes, alpha 0.05; -- for an arm with one operating point (a binary arm such as a hook setting), its false-alarm rate is not - significantly higher than A's: exact two-sided McNemar on the paired ordinary changes, alpha 0.05, with the - repository-cluster bootstrap interval of the difference reported beside it, since those changes are clustered. - -`analyse` writes `unseen.gate: pass | regression | not_run` into `result.json`. `not_run` (no slice, slice -already `informed` for this change, or more than 5% `instrument_failure`) blocks adoption like `regression`. The -fold rule and this gate must both pass. - -### 8. Increments - -1. **Used set and guard.** `eligibility.py`, the extended guard and the usage-ledger test. Exit: guard green on an - empty pool; a planted used repository, a renamed one and a fork are each rejected in tests. -2. **Sourcing and extraction.** Candidates, both change kinds and the label check, on the VM. Exit: a draft for 300 - repositories with method counts, the confirmed share and exclusion counts. -3. **Freeze p1.** Guard bisection for v3, then manifest, freeze record and protocol committed. Exit: freeze digest - committed and guard green. -4. **Dispatcher.** `benchmarks/ax/batch.py`, with the engine pass moved onto it. Exit: existing tests green and a - two-pin engine smoke on kube-ax matching its stored records. -5. **Replay task.** Image stage and entry, proven on kind, then kube-ax on five changes from this repository's own - history (never pool changes). Exit: instrument fields show bytes read and realistic JVM time, and quick-tier - output matches the VM lane. -6. **Scorer.** `score.py` on synthetic records, including every coverage state and an instrument failure. Exit: - unit tests green. -7. **Baseline.** Today's hook, from a frozen export of HEAD, on slice 1 (1,300 replays). Exit: the record committed - (counts and digests only) and a `baseline` row in the usage ledger. That is G0's exit. -8. **Arms on slice 1 (G1)** (added 2026-10-04 with decisions 4.0 and 7.3). The registered long-deadline arm, - the engine-evidence arm from the stored records, the model arms on the VM (5.4) under their budgets, and the - `unseen` block with the adoption gate in `analyse`. Exit: one committed table of arms on slice 1 at the 1/5/10% - budgets (counts, rates, intervals, digests), and `exploratory` rows in the usage ledger. - -## Error Handling - -A failed fetch, a missing commit or a hook crash gives a failure record, never an empty success. A dead sandbox is -retried once, then the VM runs it. A store or AX cleanup error stops the dispatcher. A slice with more than 5% -`instrument_failure` after re-runs is not scored; the cause is fixed first. - -## Testing Strategy - -Unit tests: used-set normalisation (case, `.git`, renames, forks), blame acceptance on a synthetic repository, -ordinary-commit exclusions, Wilson values against section 3, budget thresholds, coverage states, the task manifest -and dispatcher resume and fallback with a fake lane. Guard tests run in CI; cluster runs stay manual. - -## Requirement Traceability - -| Requirement | Design | -| --- | --- | -| 1.1 eligibility against every source, v3 via the guard | 1.2 | -| 1.2 frozen manifest, never edited | 6, increment 3 | -| 1.3 pool directory, guard extended | 6 | -| 1.4 licences, private manifest | 1.4 | -| 2.1 introducing changes, family, function, post-cutoff | 1.1, 1.3, 2.1 | -| 2.2 ordinary changes across time and size | 2.2 | -| 2.3 sizes for interval widths | 3 | -| 3.1 default replay, skipped checks kept | 4, 5.2 | -| 3.2 catch and false-alarm rules | 4.1, 4.2 | -| 3.3 per family, Wilson, coverage | 4.3, 7 | -| 3.4 arms at 1/5/10% budgets | 4.0, 7.1, increment 8 | -| 4.1 one change one Task, prefix, image by digest | 5.1, 5.2 | -| 4.2 HTTPS fetch, presigned record, no keys | 5.2 | -| 4.3 generic dispatcher, concurrency, retry, fallback, resume | 5.3 | -| 4.4 model arms on the VM within budgets | 5.4 | -| 4.5 kind runs the same tasks | 5.3, 5 (kind paragraph) | -| 5.1 baseline on slice 1, counts and digests | 6, increment 7 | -| 5.2 fresh slice per decision | 3.1, usage ledger | -| 5.3 experiments report unseen next to folds | 7.2 | -| 5.4 adoption needs no significant unseen regression | 7.3 | - -## Amendments (2026-10-11) - -Folded in after the G0 pool was frozen; the original sections above stay as the design of record, these refine it -where reality differed. - -- **Persistence is bucket-first, not local disk.** The draw streams every stage to the `ousast-corpus` RustFS - bucket and resumes from it, so a restart (even after a code change) is cheap. Layout: `unseen//decisions/` - (accept/reject per repo, key `sha256(repo+fix)`, idempotent), `unseen//resolved/` (per-advisory - resolution), `unseen//advisories/list.json`, `unseen/repos/` (get_repo cache, **not** seed-scoped, shared - across seeds), and `unseen//pool/` (the frozen pool + provenance — the "don't redo sourcing" archive). The - local journal is a within-process cursor only. Implementation: `benchmarks/unseen/bank.py` (`CorpusBank`). The - VM's local disk is the scarce resource; run artifacts and replay outputs belong on RustFS (see steering - `storage-streaming.md`). Credentials: `.env` `CORPUS_S3_*`; boto3 needs path-style + SigV4. -- **Harvest cost is GitHub-rate-limit bound, measured yield ~2.2%.** Through the strict change criteria (1 vuln + - 12 stratified ordinary, >=40 eligible commits) ~2.2% of candidates are accepted, so "100 repos" is ~4,500 - candidate attempts at ~4 GitHub calls each — the bottleneck is the 5,000/hr API limit, not CPU or cores. Pure - GitHub-API harvesting cannot replace the clone: the positive is the SZZ introducing commit and the negatives are - stratified over full history, both needing repo history. The convergence fix was a full-objects clone (dropping - `--filter=blob:none`, which caused ~600 per-blob lazy fetches) plus the bucket caches above. -- **Record 07 is a machine-label baseline.** The pool's 100 vulnerability labels are SZZ-derived and unchecked - (`freeze-p1.json` `label_check_counts`: 100 unchecked). The G0 baseline is reported as a machine-label number - with that caveat; a human label pass is a later quality refinement, required before any published precision - claim, not before record 07. -- **Replay image provenance is pinned.** CI `engine-image.yml` builds `ghcr.io/norandom/ousast-replay-task` (the - replay harness, which `COPY`s `/app/benchmarks/unseen/replay_entry.py`) and `ousast-engine-task` (the Joern - hook), publicly pullable by digest. `ousast-runner` (the 248 MB ax-task-runner, no `/app/benchmarks`, no Joern) - is **not** the replay image. The AX lane's store check (`benchmarks/ax/batch.py`) requires a public-HTTPS DNS - presign-capable endpoint; how that and the cluster's store access are provided is operator-owned infra, not - assumed here. - -- **AX lane blocker (2026-10-11) — kube is the target lane, so this is tracked, not dismissed.** The replay - *dispatches* correctly but tasks do not *complete*: `pins_done` stayed 0 for both the 5-change proof and the - 1,300-change slice 1. Sub-issues resolved along the way: (a) the replay image is `ousast-replay-task` (public); - (b) the ax CLI must be `~/go/bin/ax` — the `~/.local/ax-v0.3.1/ax` build calls an `UpdateTask` RPC the deployed - ax-server answers `Unimplemented` (client/server version mismatch); (c) the store endpoint - `files.because-security.com` / `sast-memory` passes the egress check. **Remaining blocker (AX/cluster-internal, - operator-owned):** dispatched tasks reach the `ousast-pool` workers and the replay actors execute and checkpoint, - but the result never returns to the coordinator and no completion is recorded. `ousast-pool` logs show a gVisor - cleanup failure: `runsc delete ... connecting to control server at PID NNN: connection refused` and `failed to - terminate workload after checkpoint ... runsc delete: exit status 128`. Until the completion / checkpoint-cleanup - path is fixed, the ax lane cannot produce results; the G0 baseline (7.2) was taken on the **docker lane** - instead. The ax lane must be restored before it is the baseline/G1 lane (the stated target). - -- **Engine coverage gap is comparison, not deadline (2026-10-11, preliminary n=1).** The cold-cache default arm - (30s) times out the Joern CPG build → `quick_only`. But a long-deadline sample (`--arm long_deadline`, 300s) on a - vulnerability change ran **206s with no timeout and still did not cover** (`coverage_status: incomplete`, - `quick_only`): the reasons were `ambiguous_line_correspondence`, `unmatched_added_deleted_path_correspondence`, - `eligibility_provenance_mismatch`, `witness_unresolved`, `comparison_unknown` — i.e. the engine could not - correspond the replayed diff's lines between base and head. So **calibration / a longer deadline alone will not - give engine coverage**; the blocker is the engine's base↔head comparison on real replayed diffs (consistent with - the earlier "engine near-blind to fixes" slice-leak audit). This is G2 engine-fix territory (comparison / - line-correspondence), not a G0 deadline knob. To confirm beyond n=1, the long-deadline arm must run at scale - (needs the kube lane). Implication: the record-07 baseline is measured on the **quick-rule tier** (the default - arm), and the engine arm's value is gated on fixing the comparison path, not on calibration. - -- **Line-correspondence fix made, but it is necessary-not-sufficient (2026-10-11).** Root cause confirmed in - `src/openultrasast/push/snapshot.py`: equal-region base↔head line pairs were mapped only when the line content - was *globally unique in the file*, so common lines (`}`, blanks, imports) tripped `ambiguous_line_correspondence` - and the finding's anchor was discarded → `comparison_unknown` → finding suppressed. Fixed to **run-aware - positional anchoring**: a pair maps when its content occupies exactly one contiguous equal run per side (intra-run - duplicates are forced/safe; cross-run duplicates stay withheld — soundness preserved, `test_push_snapshot.py` - moved-operation guard still passes). 163 push tests pass. BUT live-testing the fix (mounted over the installed - package, 300s) on a real messy change (a 14-file Java diff) did **not** flip coverage: it stayed `incomplete` - because the binding constraints are elsewhere — `capability_unavailable` (empty capability registry always blocks - `coverage_status=complete` until Task 8.3), the engine produced **no actionable defect at all** - (`actionable_defect_ids: []`, a detection gap, not correspondence), plus `eligibility_provenance_mismatch` and - `contributor-scan ... projection_unavailable`. So "near-blind to fixes" is **multi-causal**; the correspondence - fix removes one real cause but engine coverage on unseen diffs also needs the capability registry (Task 8.3), the - engine's actual vuln detection, and the eligibility/projection paths. Validating the correspondence fix's benefit - needs a change where correspondence is the sole blocker (a small single-file diff), or the engine arm at scale. - -## Open Questions - -Maintainer decisions of 2026-10-04 resolve questions 1, 2, 4 and 5; question 3 stays open for G3. - -1. **Resolved 2026-10-04.** Licences: only permissively licensed repositories are published by name. Copyleft and - all other licences go to the gitignored private manifest, covered in the committed manifest by an opaque id and - a digest (Req 1.4 as written; decision 1.4). Population v2's OSI-approved bar is not used. -2. **Resolved 2026-10-04.** Concurrency stays at two AX lanes plus the VM lane. No operator placement change is - requested (5.3). -3. Per-family BLOCK (G3) needs about 100 changes per family. Pool slices together, or draw a family-enriched pool - for G3? -4. **Resolved 2026-10-04.** An adoption requires no significant regression on the unseen slice, not only a report - (Req 5.4, amended; decision 7.3). -5. **Resolved 2026-10-04.** G1 adds a registered long-deadline arm (`--deadline 300`) next to the default 30 s, so - the table shows both what the user sees and what the engine could catch (decision 4.0, increment 8). diff --git a/.kiro/specs/unseen-repo-evaluation/requirements.md b/.kiro/specs/unseen-repo-evaluation/requirements.md deleted file mode 100644 index 4f9e326..0000000 --- a/.kiro/specs/unseen-repo-evaluation/requirements.md +++ /dev/null @@ -1,107 +0,0 @@ -# Requirements Document - -## Introduction - -The safety net is meant for repositories it has never seen. This spec builds the instrument that measures that: a -frozen pool of unseen repositories, a push-level metric (catch rate and false-alarm rate as the hook reports them), -and an execution path that runs the replays as AX Tasks on the kube-ax cluster. Context: `brief.md`, the roadmap's -G0-G1 (`.kiro/steering/roadmap.md`). - -## Boundary Context - -- **In scope**: pool selection and freezing; change extraction (vulnerability-introducing and ordinary commits); - replay of each change through `ousast pre-push` and through registered experiment arms; the metric and its - confidence intervals; AX Task execution with kind/VM fallback; the baseline for today's hook; rotation of pool - slices. -- **Out of scope**: any detection change; population v3 (stays reserved and unread); model calls inside the cluster. -- **Adjacent expectations**: `learned-decision-engine` experiments use this pool as their primary unseen-repo score; - `pre-push-safety-net` owns the hook being replayed; `plane-on-kubernetes` and the AX executor - (`benchmarks/learn/engine_trace_ax.py`) supply the execution pattern; the reserved-repository guard covers every - population and this pool. - -## Requirements - -### Requirement 1: A pool no development step has touched - -**User Story:** As the maintainer, I want the evaluation repositories to be ones nothing in development has seen, so a -good number means the approach generalises. - -#### Acceptance Criteria - -1. A repository is eligible only if it appears in no pair corpus, no population (v1, v2, v3), no harvest record, no - memory example, no experiment unit and no measurement record. Eligibility is checked by code against every source - before the pool is frozen; v3 is checked through the existing guard without being opened. -2. The pool is frozen (a committed manifest with repository URLs, commit SHAs, licences and the freeze digest) before - any change in it is replayed. A frozen pool is never edited; a correction is a new pool. -3. Pool repositories are referenced only inside the pool's own directory (like `benchmarks/independent/`), and the - reserved-repository guard is extended to them. -4. Only permissively licensed repositories are published by name; others may be used with their names kept out of - the repository (pointers in a local, gitignored manifest), recorded as such. - -### Requirement 2: Two kinds of change per repository - -**User Story:** As the maintainer, I want both vulnerable and ordinary pushes from the same repositories, so the -false-alarm rate is measured on the code the catch rate is measured on. - -#### Acceptance Criteria - -1. Vulnerability-introducing changes: from public advisories with a linked fix, the commit range that introduced the - vulnerable code where it can be determined, else the fix's parent compared with an earlier base; each labelled - with family and the vulnerable function. Post-model-cutoff advisories are preferred and flagged. -2. Ordinary changes: commits from the same repositories' histories with no security fix, revert or advisory link, - sampled across time and size; each recorded with its changed files and lines. -3. Per repository, at least one vulnerability-introducing change and several ordinary changes; the pool's totals are - sized so the false-alarm rate's 95% interval is no wider than +/- 3 points and the catch rate's no wider than - +/- 10 points (sizes computed in the design). - -### Requirement 3: The push-level metric - -**User Story:** As the maintainer, I want the score to be what a user would see, so the number means something. - -#### Acceptance Criteria - -1. Each change is replayed as `ousast pre-push --base --head ` with the hook's default settings, in a - clean checkout, with the record of every skipped check kept. -2. Catch: a vulnerability-introducing change counts as caught when the hook (or the arm) reports a finding of the - right family on the vulnerable function or its changed lines. False alarm: an ordinary change counts as a false - alarm when the hook reports any advisory finding. -3. Reported per family and overall: catch rate, false-alarm rate, and both with 95% Wilson intervals; plus coverage - (changes where the hook could analyse the changed language) so a silent zero is visible. -4. Arms (decision engine, combiner, engine evidence, fix mechanisms) are scored on the same replays: catch rate at - fixed false-alarm budgets (1%, 5%, 10%), so arms are compared at equal noise. - -### Requirement 4: Execution on kube-ax, designed for AX - -**User Story:** As the maintainer, I want the replays to run on the cluster, so measurements are fast and the VM is -free. - -#### Acceptance Criteria - -1. One replayed change is one AX Task, named with the `ousast-engine-` prefix the operator's egress watcher matches, - using an image built `FROM ghcr.io/norandom/ax-task-runner:v0.3.1` with this package, Joern and php-cli, pinned by - digest. -2. The task fetches the repository at the needed commits itself over HTTPS (public hosts are reachable from actors) - and writes its record through a presigned link; no API keys or store credentials enter the cluster. -3. A generic AX batch dispatcher serves both this replay and the existing engine passes: items to Tasks, at most the - cluster's safe concurrency (two heavy tasks per worker), retry once on sandbox death, then a VM or kind fallback - lane; progress and resume as in the engine runner. -4. Arms that need model calls take the cluster's records and make their calls on the VM, within registered budgets. -5. kind runs the same tasks unchanged as the local fallback. - -### Requirement 5: Baseline, rotation and use - -**User Story:** As the maintainer, I want a baseline now and a fresh slice for each decision, so the pool is not -worn out by tuning. - -#### Acceptance Criteria - -1. The first use is a baseline of today's hook on the first slice; its record is committed (counts and digests only, - no repository names outside the pool directory). -2. A slice that informed a change can no longer qualify that change; each adoption decision draws a new, untouched - slice, recorded with its freeze digest. -3. Every learned-decision-engine experiment from now on reports its unseen-repo score from this pool next to its - held-out-fold score. -4. An arm is adopted only if, on the untouched slice that qualifies it, it shows no significant regression against - the current production arm: catch at the 5% false-alarm budget not significantly lower, and for an arm with one - operating point the false-alarm rate not significantly higher (exact paired tests, two-sided, alpha 0.05). - Reporting the unseen score alone is not enough to adopt. *(amended 2026-10-04, maintainer decision)* diff --git a/.kiro/specs/unseen-repo-evaluation/spec.json b/.kiro/specs/unseen-repo-evaluation/spec.json deleted file mode 100644 index 9418b06..0000000 --- a/.kiro/specs/unseen-repo-evaluation/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "feature_name": "unseen-repo-evaluation", - "created_at": "2026-10-04T18:00:00Z", - "updated_at": "2026-10-04T20:00:00Z", - "language": "en", - "phase": "implementation", - "approvals": { - "requirements": { - "generated": true, - "approved": true - }, - "design": { - "generated": true, - "approved": true - }, - "tasks": { - "generated": true, - "approved": true - } - }, - "ready_for_implementation": true -} diff --git a/.kiro/specs/unseen-repo-evaluation/tasks.md b/.kiro/specs/unseen-repo-evaluation/tasks.md deleted file mode 100644 index 9aa822f..0000000 --- a/.kiro/specs/unseen-repo-evaluation/tasks.md +++ /dev/null @@ -1,402 +0,0 @@ -# Implementation Plan - -One task group per increment of `design.md` section 8, in that order. A group starts only after the previous -group's exit is recorded. Maintainer decisions of 2026-10-04 are folded in: a registered long-deadline arm -(`--deadline 300`) beside the hook's default 30 s (design 4.0, group 8); adoption needs no significant regression -on the unseen slice (Req 5.4 as amended, design 7.3, tasks 6.4 and 8.2); only permissively licensed repositories -are published by name, everything else goes to the gitignored private manifest under a digest (Req 1.4, design -1.4); concurrency stays at two AX lanes plus the VM lane (design 5.3). - -**2026-10-11 amendments folded in** (see design.md "Amendments (2026-10-11)"): - -- **Pool p1 is frozen.** `benchmarks/unseen/pool-p1.toml` = 100 repos, slice 1; `freeze-p1.json` guard=0, - dropped=0, deterministic (manifest `6bd2fa5d`); archived to `ousast-corpus/unseen/20260601/pool/`. Groups 1-3 are - effectively done (the draw, draft and freeze ran end to end); formal records 02/03 were not written as separate - `record.json` files — the freeze was run directly (`bisect_guard` then `freeze`). Freeze recipe: `bisect_guard - --root $PWD --draft $CACHE/journal/draft-p1.toml` then `freeze --root $PWD --used-report ` where the file - is the draw report's `used_set` object (not the whole report). -- **Persistence is bucket-first** (`ousast-corpus`, `benchmarks/unseen/bank.py`); the local journal is a - within-process cursor only. Raw per-change replay records go to **RustFS**, not `~/ousast-results/` on local disk - — amend tasks 5.3/7.2's `--out` accordingly. -- **Group 5 image note:** the replay image is the CI-built `ghcr.io/norandom/ousast-replay-task` (+ engine-task), - publicly pullable by digest — not a local export-HEAD build and not `ousast-runner`. Task 7.1's "build and push - the replay image" is satisfied by the CI `engine-image.yml` run; use that digest. -- **Record 07 is a machine-label baseline** (labels SZZ-derived, 100 unchecked); state the caveat in the record. -- **Baseline lane (7.2):** run on kube (`--lane ax`). The `ax` binary and the AX-lane store endpoint are - operator-owned infra to stand up before 7.2; do not assume the cluster/network state. - -Execution rules: - -- **Who does what.** Codex implements every coding task offline: no network, no model, no store, no cluster. The - coordinator runs every networked step (GitHub and OSV API calls, git fetches and clones, the S3 store, image - pushes, kind and kube-ax). Those tasks are marked **(coordinator, network)**. Steps that need the coordinator but - no network (agent label reads, adjudication, the v3 guard bisection, commits) are marked **(coordinator)**. - Steps that spend model money are marked **paid: needs a budget go-ahead**. Each starts only after the maintainer - confirms a ceiling against the provider balance read just before it. -- **Done means.** A coding task is done when its tests pass in the host suite, `ruff check`, `ruff format --check` - and `mypy` are clean, and the reserved-repository guard is green. No test makes a network, model or cluster call. -- **Names stay in the pool directory.** Pool repository names and URLs appear only under `benchmarks/unseen/` - (copyleft and other non-permissive ones only under the gitignored `benchmarks/unseen/private/`). Raw per-change - records live in `~/ousast-results/unseen/` and the store, never in the repository. -- **Population v3 is never opened.** It is checked only through the guard test, run as a black box with its - output discarded. -- **Every group ends with a measured exit** and a record at - `benchmarks/measurements/-unseen-NN-/record.json`: commit, image digests where used, the instrument - checks (bytes read, JVM time, exit codes), counts and digests. No repository names. The guard sweeps these - records. -- **Instrument first** (AGENTS.md). A zero, an empty draft or a fast run is treated as an instrument failure until - bytes read and wall time prove otherwise. - -- [x] 1. Used set and guard (design 1.2, 6; increment 1) -- [x] 1.1 Used-set builder - - `benchmarks/unseen/eligibility.py`: normalises names (lower case, no `.git`, no trailing slash) and builds the - used set from every source of design table 1.2: pair catalogs, recipes and datasets under `benchmarks/pairs/` - plus the local pointer-pair manifests as `benchmarks/pairs/catalog_gen.py` reads them; `population-v1.toml` - and `population-v2.toml`; the advisory-fix batch catalogs; memory `example` rows (`repo` field) through an - injected `MemoryStore` (`FileStore` locally, the S3 store when the coordinator passes one); experiment units - under `plane/experiments/`; `benchmarks/measurements/` and `benchmarks/experiments/`; a text sweep for - `github.com//` over `benchmarks/`, `src/`, `tests/`, `plane/` and `.kiro/`. It never reads - population v3 and never prints a name. Output: counts per source and the sha256 of the sorted set. - - `tests/test_unseen_eligibility.py`: a synthetic tree with one repository per source, each found; case, `.git` - and trailing-slash variants collapse; stdout and the output JSON contain no name; a v3-shaped file planted in - the synthetic tree is not opened (open is patched to fail on it). - - _Requirements: 1.1_ -- [x] 1.2 Canonical resolution, renames and forks - - `eligibility.py`: `resolve(candidate, api)` with an injected GitHub client returns the canonical `full_name`, - former names (redirects) and the fork network (`parent`, `source`). A candidate is ineligible when any of them - is in the used set; one candidate per fork network is kept. - - Tests with a fake client: a planted used repository, a renamed one and a fork of a used one are each - rejected; two forks of one network keep one; the rejection reason is a source label, not a name. - - _Requirements: 1.1_ -- [x] 1.3 Guard extended to pools - - `tests/test_independent_population.py`: the sweep becomes a list of (reservation directory, names): - `benchmarks/independent/` for populations, `benchmarks/unseen/` for pools (public manifests, plus - `benchmarks/unseen/private/*.toml` when present). The sweep also covers `plane/`. New tests: pool and - population names are disjoint (failure prints a count only); a frozen pool's manifest digest equals its - `freeze-pN.json` (skipped while no pool is frozen). The existing v3 handling is unchanged. - - Each reservation is exempt only inside its own directory: population names (v3 included) are still swept in - `benchmarks/unseen/`, the gitignored `private/` files included. Task 3.1 depends on this. - - `.gitignore`: `benchmarks/unseen/private/`. - - Evidence: the guard is green with an empty `benchmarks/unseen/`, and red when a test fixture names a pool - repository in `benchmarks/measurements/` (shown once in the record). - - _Requirements: 1.2, 1.3, 1.4_ -- [x] 1.4 Usage ledger - - `benchmarks/unseen/ledger.py`: append-only `usage-pN.jsonl`, one row per use (slice, decision or experiment - id, purpose `baseline | exploratory | informed | qualifies`, date, result digest, freeze digest). It refuses a - `qualifies` row for a change whose slice already has an `informed` row for it, and refuses to rewrite rows. - - `tests/test_unseen_ledger.py`: append, the refusal, a rewritten row detected by a digest chain. - - _Requirements: 5.2_ -- [x] 1.5 Used-set measurement and record 01 **(coordinator, network)** - - Run `eligibility.py` against the real sources, the S3 memory store included (read only). - - Instrument: each source reports files read and bytes; a source with zero rows fails the run unless it is - known empty. - - `benchmarks/measurements/-unseen-01-used-set/record.json`: counts per source, used-set digest, guard - result (green on the empty pool), test counts of 1.1-1.4. - - _Requirements: 1.1, 1.3_ - -- [ ] 2. Sourcing and extraction (design 1.1, 1.3, 1.4, 2, 3; increment 2) -- [x] 2.1 Advisory candidates and repository filters - - `benchmarks/unseen/source.py`: from GHSA advisory JSON (pip, npm, maven, composer; published 2019 or later) - keep those whose CWE maps to a family through `src/openultrasast/model/taxonomy.py` and that link one fix - commit with one parent in one repository; attach the OSV first affected version; flag `post_cutoff` when - published after 2026-06-01. Repository filters: anonymous HTTPS, at most 500 MB, licence at the pinned head - (LICENSE file, then the API's SPDX id) classed `permissive` (the design 1.4 list) or `private`. - - `tests/test_unseen_source.py` on recorded fixture JSON (no network): CWE mapping, multi-parent and - multi-repository fixes dropped, cutoff flag, licence classing (GPL, AGPL, LGPL, none and source-available go - private). - - _Requirements: 1.4, 2.1_ -- [x] 2.2 Vulnerability-introducing range - - `benchmarks/unseen/extract.py`: the `introducing` method of design 2.1 (blame of the sink line, or the - declaration for an absence bug; absence at V's first parent; no merge or root; at most 50 files and 2,000 - lines; OSV version bracket when present; walk-back of at most five steps) and the `last_touch` fallback. Each - change records method, family, vulnerable function resolved in head, head-side lines, `post_cutoff`, advisory - id. Git only; no scanner or engine. - - `tests/test_unseen_extract.py` on a synthetic repository built in a temp dir: accepted V; a reformat walked - back; a merge rejected; size cap; bracket rejection; fallback chosen when no V passes. - - _Requirements: 2.1_ -- [x] 2.3 Ordinary-commit draw - - `extract.py`: first-parent, non-merge commits touching an analysed language; the exclusions of design 2.2 - (security words of the narrow pattern, reverts, bot and bump commits, manifest- or lock-only, generated or - vendored, advisory fix commits, commits touching a known vulnerable function, more than 1,000 source lines, - counted). Seeded draw of 12 per repository over three time thirds by four size buckets, proportional, at least - one per third; a repository with fewer than 40 eligible commits is dropped. - - Tests: each exclusion; the narrow pattern keeps "query", "token", "session"; allocation and the seed - reproduce the same draw; the 40-commit floor. - - _Requirements: 2.2, 2.3_ -- [x] 2.4 Draft manifest and slices - - `benchmarks/unseen/draft.py`: writes `benchmarks/unseen/draft-p1.toml` (permissive entries by name) and - `benchmarks/unseen/private/draft-p1.toml` (the rest); the public file holds an opaque id and - sha256(URL, commits) per private entry. Seeded shuffle into three slices of 100 repositories, stratified by - ecosystem and `post_cutoff`; each repository has one introducing change and 12 ordinary changes. - - Tests: the digest covers private entries; no private URL in the public file; the same seed gives the same - slices; per-slice strata within one repository of proportional. - - _Requirements: 1.2, 1.4, 2.3, 5.2_ -- [ ] 2.5 Sourcing run **(coordinator, network)** - - `gh api /advisories` enumeration, OSV lookups, canonical resolution against the used set (1.2), blobless - clones into a scratch cache, extraction (2.2, 2.3), draft (2.4). - - Instrument: clones counted with bytes on disk; a repository whose clone reads zero source files is an - instrument failure, not a drop. Fewer than 300 eligible repositories stops the run with counts; the - eligibility bars are not relaxed to fill it. - - Observable: a draft of 300 repositories (3,900 changes) on disk. - - _Requirements: 1.1, 1.4, 2.1, 2.2, 2.3_ -- [ ] 2.6 Label check and record 02 **(coordinator)** - - Every third `introducing` change read by an agent, no model call, marked `confirmed | refactor | unclear` - in the draft. - - `benchmarks/measurements/-unseen-02-draft/record.json`: candidates per ecosystem, rejections per used - source, licence split (public, private), method counts, post-cutoff counts, confirmed share, ordinary-commit - exclusion counts. No names. - - _Requirements: 2.1, 2.2_ - -- [ ] 3. Freeze p1 (design 1.2, 6; increment 3) -- [x] 3.1 Guard bisection tool - - `benchmarks/unseen/bisect_guard.py`: rewrites the draft with a candidate subset, runs - `.venv/bin/pytest -q -p no:cacheprovider tests/test_independent_population.py -k referenced` with stdout and - stderr discarded, reads only the exit code, and bisects until green. Reports the dropped count only. - - `tests/test_unseen_bisect.py` with an injected fake guard (a hidden set): finds and drops exactly the hidden - members; never logs a candidate; an exit code other than 0 or 1 aborts (instrument failure). - - _Requirements: 1.1_ -- [x] 3.2 Freeze writer - - `benchmarks/unseen/freeze.py`: canonical `pool-p1.toml` (and the private counterpart) from the bisected draft; - `freeze-p1.json` with used-set digest and per-source counts, guard result, dropped count, label-check counts - and the freeze digest (sha256 of the canonical manifest). Refuses to write over a frozen pool; a correction - is `p2`. - - Tests: digest stable under key order; overwrite refused; the guard's digest test (1.3) passes on a fixture - pool. - - _Requirements: 1.2_ -- [x] 3.3 Protocol p1 - - `benchmarks/unseen/protocol-p1.md`: the scoring rules of design section 4 (default flags, decision 4.0's - long-deadline arm, catch and location rules, false alarm, coverage states), the statistics of section 3 and - 7, and the adoption gate of 7.3, stated before any replay. - - Observable: the file exists and its sha256 is listed in `freeze-p1.json`. - - _Requirements: 3.1, 3.2, 3.3, 3.4, 5.4_ -- [ ] 3.4 Freeze run and record 03 **(coordinator)** - - Run the bisection (3.1), then the freeze (3.2). Guard green, digest test active. - - `benchmarks/measurements/-unseen-03-freeze/record.json`: freeze digest, protocol digest, dropped - count, final counts per slice and kind. - - _Requirements: 1.1, 1.2, 1.3_ - -- [ ] 4. Dispatcher (design 5.3; increment 4) -- [x] 4.1 Items, workloads and the AX lane - - `benchmarks/ax/batch.py`: `Item` (id, input objects, result object name, command, extra env, input digest), - `Workload` (image digest, allowed URL env names, deadline, result validator) and the `ax` lane, moved from - `benchmarks/learn/engine_trace_ax.py`: manifest render, egress readiness, presigned GET and PUT as `.dat`, - apply, poll result object and AX phase, delete, object cleanup. The `kind` lane is the same lane with the - kind context. - - `tests/test_ax_batch.py` with a fake `ax` CLI and a fake store: manifest holds only the allowed URL env - names and no credential; names carry the `ousast-engine-` prefix; cleanup on success and failure. - - _Requirements: 4.1, 4.2, 4.5_ -- [x] 4.2 Scheduler, fallback and resume - - `batch.py`: two AX lanes and one VM `docker` lane by default (one container at a time); sandbox death retried - once under a new name, a second death or a JVM OOM goes to the VM fallback queue ahead of overflow; - `progress.json` written by the scheduler only; a `STOP` file halts launches; resume skips items whose stored - record has the same input digest; a cleanup failure stops dispatching. - - Tests with fake lanes: retry then fallback; OOM straight to fallback; resume skips; `STOP`; cleanup failure - halts; never more than two AX tasks in flight. - - _Requirements: 4.3_ -- [x] 4.3 Engine pass on the dispatcher - - `benchmarks/learn/engine_trace_ax.py` and `engine_trace.run_plan` become an engine `Workload` over `batch.py`; - no behaviour change. - - Observable: `tests/test_ax_executor.py` and `tests/test_engine_trace.py` pass with their assertions unchanged - (moved imports only). - - _Requirements: 4.3_ -- [ ] 4.4 Engine smoke on kube-ax and record 04 **(coordinator, network)** - - Two pins through the new dispatcher on kube-ax, compared with their stored records. - - `benchmarks/measurements/-unseen-04-dispatcher/record.json`: image digest, per-pin wall time and JVM - time, record digests equal to the stored ones (or the diff counted), cleanup result. - - _Requirements: 4.3_ - -- [ ] 5. Replay task (design 5.1, 5.2; increment 5) -- [x] 5.1 Image stage - - `plane/Dockerfile.engine-task`: second stage `FROM engine AS replay` adding `git`, the package with its - runtime dependencies and `benchmarks/unseen/replay_entry.py`; smoke `ousast pre-push --help`. The root image - stays `ghcr.io/norandom/ax-task-runner:v0.3.1`; the engine stage is unchanged. - - `tests/test_unseen_image.py`: parses the Dockerfile; the `replay` stage builds from `engine`; the root image - pin; the engine stage's text is byte-identical to before. - - _Requirements: 4.1_ -- [x] 5.2 Task entry - - `benchmarks/unseen/replay_entry.py`, standard library only, reusing `engine_task_entry.transfer`: GET the spec; - `git init` and depth-1 fetches of head and base over HTTPS (180 s bound), verified with `git cat-file -e`; run - `ousast pre-push` with the spec's hook flags (defaults, or `--deadline 300` for the long-deadline arm), heap - 1700 MB and the cluster JVM flags; pack `push.json`, `push.txt`, `engine.json`, `instrument.json` (files and - bytes in the checkout, bytes of each changed file, JVM wall time, hook exit code, step timings, image - digest); PUT as `.dat`. Any error PUTs a failure record naming the step and status, never URL text. - - `tests/test_unseen_replay_entry.py`: a local bare repository and a localhost presign stub; success record - fields; a missing commit, a failed fetch and a hook crash each give a failure record; zero bytes read and a - replay faster than JVM start are flagged `suspect_fast`. - - _Requirements: 3.1, 4.1, 4.2_ -- [x] 5.3 Replay workload - - `benchmarks/unseen/replay.py`: builds `Item`s from a frozen slice and an arm (`default` or `long_deadline`); - the spec object carries the repository URL so names stay out of AX status and telemetry; task names - `ousast-engine-replay--` with an opaque slug; the validator rejects a record without - `instrument.json`; records go to `~/ousast-results/unseen////` and the store. - - Tests: no repository name in any task name or manifest; only the two presigned URLs in the env; input - digest changes with the arm's flags; `instrument_failure` items queued for one re-run. - - _Requirements: 3.1, 4.1, 4.2, 4.3_ -- [ ] 5.4 Proof on kind, then kube-ax, and record 05 **(coordinator, network)** - - Build and push the `replay` target, pinned by digest. Five changes from this repository's own history (never - pool changes) on kind, then the same five on kube-ax, then on the VM lane. - - Exit: instrument fields show bytes read and realistic JVM time on every lane; quick-tier findings equal - across lanes. - - `benchmarks/measurements/-unseen-05-replay-task/record.json`: image digest, per-lane timings, bytes, - exit codes, the quick-tier comparison. - - _Requirements: 4.1, 4.2, 4.5_ - -- [x] 6. Scorer (design 3, 4, 7.1, 7.3; increment 6) -- [x] 6.1 Coverage states - - `benchmarks/unseen/score.py`: each record gets one state (`instrument_failure`, `unanalysable`, `quick_only`, - `engine_covered`) from `instrument.json` and the artifact; `suspect_fast` goes to failure; a slice above 5% - `instrument_failure` after re-runs is refused. - - `tests/test_unseen_score.py` on synthetic records: every state; the refusal; a failure never counts as quiet - or missed. - - _Requirements: 3.3_ -- [x] 6.2 Catch and false-alarm rules - - `score.py`: family match (engine family, quick-rule CWE through `taxonomy.py`), the location table of design - 4.1, all three streams with the stream named per catch; false alarm on any finding of any stream; the - alerts-only rate. - - Tests: right family wrong place misses; `access_control` on the handler; `config_secrets` file scope; a - quick-rule-only catch; one advisory finding on an ordinary change is a false alarm. - - _Requirements: 3.2_ -- [x] 6.3 Rates, intervals and breakdowns - - `score.py`: Wilson 95% intervals; repository-cluster bootstrap for false alarms; ICC; breakdowns per family, - method, `post_cutoff`, stream, coverage state and the confirmed subset. - - Tests: the n values of design section 3 (315, 483, 756, 62, 88, 97) reproduced; bootstrap seeded; ICC on a - synthetic clustered set. - - _Requirements: 2.3, 3.3_ -- [x] 6.4 Budgets, paired tests, the gate and record 06 - - `score.py`: per-budget thresholds (1%, 5%, 10%), labelled `exploratory` in-slice and `fixed` when supplied; - a binary arm's single operating point with the budgets it meets; exact two-sided McNemar for catch and for - false alarm; `gate(A, B)` returning `pass | regression | not_run` per design 7.3. - - Tests: thresholds on a synthetic score set; McNemar exact values; a binary arm with a significant - false-alarm increase gives `regression`; missing slice or excess failures give `not_run`. - - `benchmarks/measurements/-unseen-06-scorer/record.json`: test counts and the digest of the scorer's - output on the committed synthetic fixture. - - _Requirements: 3.4, 5.4_ - -- [ ] 7. Baseline (design 4, 6; increment 7; G0 exit) -- [ ] 7.1 Frozen source and dry run **(coordinator, network)** - - Export HEAD to a frozen tree; build and push the `replay` image from it, pinned by digest. - - `benchmarks/push/python_overhead.py` run with the stub engine to size per-replay Python cost; dispatcher dry - run on slice 1 lists 1,300 items without launching. - - _Requirements: 5.1, 4.1_ -- [ ] 7.2 Baseline replay of slice 1 **(coordinator, network)** - - Default arm, 1,300 replays over two AX lanes and the VM lane (about 9 hours); `instrument_failure` items - re-run once; the run stops for a cause above 5% failures. - - Observable: 1,300 records in `~/ousast-results/unseen/p1/1/default/` and the store, each with - `instrument.json`. - - _Requirements: 3.1, 4.3, 5.1_ -- [ ] 7.3 Adjudication **(coordinator)** - - Flagged ordinary changes adjudicated against protocol-v2's standard (all if 30 or fewer, else every third); - the adjudicated share kept beside the false-alarm rate. ICC measured; above 0.05 noted as "next pool sizes - up". - - _Requirements: 3.2, 2.3_ -- [ ] 7.4 Record 07 and the ledger **(coordinator)** - - `benchmarks/measurements/-unseen-07-baseline/record.json`: `score.py` output (counts, rates, Wilson and - bootstrap intervals, coverage per state, per family, method, stream), freeze digest, image digest, commit. - `usage-p1.jsonl` row `baseline`. - - Observable: guard green over the record; G0's exit met. - - _Requirements: 5.1, 5.2, 3.3_ - -- [ ] 8. Arms on slice 1 (design 4.0, 5.4, 7.1-7.3; increment 8; G1) -- [ ] 8.1 Registered long-deadline arm - - `plane/experiments/` manifest for the long-deadline arm: hook flags with `--deadline 300`, budget 0, - `unseen: {pool: p1, slice: 1}`, committed before its first replay. `src/openultrasast/learn/experiments.py` - accepts `unseen` and a hook-flag arm. - - `tests/test_learn_experiments.py`: the manifest loads; an `unseen` block without pool or slice is refused. - - _Requirements: 3.4, 5.3_ -- [ ] 8.2 `unseen` block and adoption gate in `analyse` - - `experiments.py` `analyse`: writes `unseen` into `result.json` (catch at each budget per arm, paired B - A, - McNemar, `gate`) from `score.py`; checks the usage ledger; adoption requires the fold rule and - `gate == pass`. - - Tests: `regression` and `not_run` each block adoption; an `informed` slice cannot qualify; result JSON shape. - - _Requirements: 5.2, 5.3, 5.4_ -- [ ] 8.3 Engine-evidence arm - - `benchmarks/unseen/arms.py`: a score per change from `engine.json` (witness rows on head-side changed - functions), no model; plugs into the budget thresholds of 6.4. - - Tests on synthetic records: score monotone in witness strength; a change without engine coverage gets no - score and is reported, not counted quiet. - - _Requirements: 3.4_ -- [ ] 8.4 Model-arm adapter - - `arms.py`: candidates from head-side changed functions of stored records, scored by the current decision - engine program (and the combiner when its program exists; else its row reads `not_built`); change score is the - maximum; metered client, dry run first, response cache, the manifest's `budget_usd` as the hard ceiling. - - Tests with a scripted client: dry-run estimate, ceiling stop, cache replay; no model call. - - _Requirements: 4.4, 3.4_ -- [ ] 8.5 Long-deadline replay of slice 1 **(coordinator, network)** - - 1,300 replays of the `long_deadline` arm, bounded at about 345 s each (up to about 42 hours over three - lanes), with `STOP` and resume across sessions; failures re-run once. - - Observable: 1,300 records under `~/ousast-results/unseen/p1/1/long_deadline/`, JVM time in each. - - _Requirements: 3.1, 3.4, 4.3_ -- [ ] 8.6 Model arms on slice 1 **(coordinator, network; paid: needs a budget go-ahead)** - - Dry run gives the candidate count; estimate at the injection slice's measured cost (about $0.008 per - candidate at k = 5); the maintainer sets the ceiling against the balance read just before. Then the decision - engine arm (and the combiner if built) on the VM. - - _Requirements: 4.4, 3.4_ -- [ ] 8.7 Record 08: the arms table **(coordinator)** - - `benchmarks/measurements/-unseen-08-arms/record.json`: per arm (hook 30 s, hook 300 s, engine evidence, - decision engine, combiner or `not_built`) catch at 1/5/10% budgets with Wilson intervals and realised - false-alarm rates, coverage per state, spend; `exploratory` rows in `usage-p1.jsonl`. - - Observable: one committed table of arms on slice 1; G1's table exists. - - _Requirements: 3.4, 5.2, 5.3_ - -## Implementation Notes - -- 2026-10-04: Tasks 1.1–1.4 implemented offline and independently reviewed. Task 1.5 remains for the - coordinator; no networked measurement or commit was performed. Command and API contracts: - `benchmarks/unseen/README.md`. Offline evidence: - `benchmarks/measurements/2026-10-04-unseen-01-offline/record.json`. -- Population guard scope is preserved under the maintainer's hard rule: benchmarks/src/tests for - populations, plus plane for pools. Each directory exemption is reservation-specific, including private - pool manifests. Extending population scope to plane exposed 30 existing references; no historical - records were changed or exempted inside the existing population scope. -- S3 inventory opens through `open_store(read_only=True)` because ordinary store construction writes a - Select capability probe. Default store behavior is unchanged; inventory uses LIST/GET only. -- The freeze writer must match the canonical TOML digest encoding documented in the pool README and - exercised by `test_independent_population.py`; final pool discovery does not trust mutable status. - -### Group 2 implementation evidence (2026-10-04) - -Tasks 2.1–2.4 are implemented offline and independently reviewed. Task 2.5 is -`python -m benchmarks.unseen.draft --root "$PWD" --s3-store s3:// --seed 20260601` -with the documented PATH/PYTHONPATH and GH_TOKEN environment. No networked draw, -label check, group-2 measurement record, guard bisection, or commit was performed. -The group header remains pending its coordinator exits 2.5/2.6. - -The command rebuilds the group-1 used set, resolves renamed/fork repositories, -uses disposable external blobless clones, re-reads licences at extracted heads, -and emits counts/digests only. The README documents injected clients, method bias, -the hard 300-repository floor, and an explicitly unmeasured resource estimate. -Independent review caught and fixed ambiguous parent-function absence and stale -candidate-404 handling; regression tests cover both. - -Validation: requested pytest selection 103 passed, 1 skipped, exit 0 (population-test -output discarded); `ruff check .` exit 0; `ruff format --check .` exit 0 (411 files); -`mypy src/openultrasast` exit 0 (160 source files). The CLI `--help` smoke exited 0. -Source-read instrumentation printed 20,101 bytes for the extraction implementation. - -### Groups 4–6 offline implementation (2026-10-04) - -Tasks 4.1–4.3, 5.1–5.3 and 6.1–6.4 are implemented and independently reviewed. -Tasks 4.4 and 5.4 remain coordinator-owned network proofs; group headers 4 and 5 stay -pending those exits. No commit, network request, Docker, AX or kubectl command was run. -The sourcing job's draft.py, source.py, extract.py and journal.py are unchanged. - -The shared dispatcher retains the engine adapter's historical questions.json object -name to preserve its unchanged assertions; replay uses spec.dat and result.dat. -The engine Dockerfile body is unchanged except its required AS engine alias; CI builds -engine and replay separately. S3Store's read_only class default preserves existing -constructor-bypassing transport fixtures; initialized read-only stores still refuse writes. - -Exact coordinator commands and input contracts are in benchmarks/unseen/README.md, -including frozen TOML/private pointer verification, scoring joins, own-history proof -preparation and anonymous measurement-record generation. JVM wall time is the hook's -aggregate scan build/query timing and is labelled as such. Unsupported-only partitions -with explicit language_not_covered evidence do not pretend to have launched a JVM. - -Record 06 contains the committed synthetic fixture and scorer-output digests. The -reserved-repository guard was run as a black box with both output streams discarded. -Independent review: APPROVED; all reported integration and coverage defects resolved. - -Final verification: requested pytest command 279 passed, 8 skipped, exit 0; separate -`tests/test_ax_batch.py` 5 passed, exit 0; `ruff check .`, `ruff format --check .`, -`mypy src/openultrasast`, and the output-discarded reserved-repository guard all exit 0. -The own-history proof preparer and replay dry run read five items and exit 0. diff --git a/.kiro/steering/overview.md b/.kiro/steering/overview.md index da2dde8..6804aee 100644 --- a/.kiro/steering/overview.md +++ b/.kiro/steering/overview.md @@ -1,6 +1,35 @@ # Overview — what this is, and what is actually true of it -## Current state, 2026-10-04 +## Current state, 2026-10-13 + +Everything below this section is dated history; where it disagrees with this section, this section wins. + +- **Goal, unchanged.** A pre-push safety net that works on repositories it has never seen, judged by recall on + real vulnerability changes and false alarms per 100 ordinary pushes, measured on a valid benchmark. +- **The valid benchmark exists and ran** (`valid-detection-benchmark`, `benchmarks/valid/`, pilot frozen at + `benchmarks/valid/pilot/`; numbers in `benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md`). + 12 verified fix-site positives, 240 ordinary pushes, 12 unseen app repos. Push hook as shipped: 0 of 12, + 0.7 false alarms per 100. LLM hunter: 5 to 6 of 12, 54 false alarms per 100 after the classifier. Neither + meets the gate. The SZZ pool (`pool-p1`) and its spec are retired; no number from it is quoted. +- **What the numbers say.** The hook has no LLM detection tier and catches nothing on unseen code. The hunter + finds about half but searches whole files, so it re-reports old code on every push (250 of 343 "vulnerable" + suspicions on ordinary pushes sit more than 10 lines from the diff). Thresholds on that output are not a fix. +- **Active work, in order.** (1) `hunter-learning-loop`: an annotation model for the hunter (diff hunks, + enclosing function and callers, entry points, roles, framework facts, closed families) chosen by statistical + ablation under the learn harness; requirements written 2026-10-13, design next. (2) `pre-push-safety-net`: + the LLM tier on pre-push once the search is good. (3) `search-with-proof` / `proof-arm-opencode`: proof on + what survives the search, as the block gate. (4) `engine-change-scoped-analysis`: the deterministic facts + that feed the hunter's annotations. +- **Everything else is history or parked.** Retired spec directories were removed from the tree on 2026-10-13 + (`git show :.kiro/specs//` still has them): `harnessx-removal` (51d064e), + `harnessx-self-improving-rulesets` and `openrouter-sast-harness` (c8d2fef), `learning-harness`, + `constrained-detector`, `three-stage-scan`, `real-world-vfc-slice` (39a1873), `unseen-repo-evaluation` + (abb2058). The learn harness code (`src/openultrasast/learn/`) stays; `hunter-learning-loop` builds on it. +- **Rules that stay in force.** Plain English in every document, commit and comment. Standard metrics only. + Prove the instrument before believing a zero. Train on one set, report on another. Research runs locally; + deployed behaviour is observed, not re-measured by hand. + +## Current state, 2026-10-04 (superseded by the section above) Everything below this section is dated history; where it disagrees with this section, this section wins. The direction lives in `roadmap.md`, section of 2026-10-04 (phases G0-G5). @@ -217,14 +246,14 @@ Counted 2026-10-02 from each spec's `tasks.md` checkboxes and `spec.json`. | `pre-push-safety-net` | implementation | 67 of 79 tasks done | | `contributor-scan` | paused | 27 of 51 tasks done; paused since 2026-09-23 (`v1.2.0-alpha.1` NO-GO), waits on pre-push M2/M3 | | `ai-service-plane` | complete | 11 of 11; the ax plane is the agentic path | -| `harnessx-removal` | complete | 8 of 8; HarnessX removed 2026-09-30 | +| `harnessx-removal` | complete, directory removed 2026-10-13 | 8 of 8; HarnessX removed 2026-09-30 | | `finding-feedback-loop` | dormant | requirements approved, design unapproved since 2026-09-09; depends on `contributor-scan` | | `model-grounded-detection` | complete | 16 of 16; built the arbiter this all rests on | | `authorization-obligations` | dormant | tasks approved, 0 of 24 started; superseded in practice by the dominance arbiter | | `reachability-flow-model` | dormant | 23 tasks generated, nothing approved; largely delivered by adopting Joern | | `flow-aware-ranking`, `zero-with-a-reason`, `ir-consolidation` | dormant | brief only, unapproved | | four others | complete | `pair-corpus-honesty`, `corpus-seeded-mechanisms`, `propose-adjudicate-prove`, `tree-sitter-overlay-extra` | -| six others | retired or superseded | `openrouter-sast-harness`, `three-stage-scan`, `harnessx-self-improving-rulesets`, `learning-harness` (retired with HarnessX), `constrained-detector`, `real-world-vfc-slice` (superseded) | +| six others | retired or superseded, directories removed 2026-10-13 | `openrouter-sast-harness`, `three-stage-scan`, `harnessx-self-improving-rulesets`, `learning-harness` (retired with HarnessX), `constrained-detector`, `real-world-vfc-slice` (superseded) | > **Retired 2026-09-30:** HarnessX was removed by `harnessx-removal`; the agentic plane is ax (`ai-service-plane`), > and evolutionary proposals come from the plane memory. The HarnessX specs in the table above are history. diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 61a185c..f158b07 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1473,13 +1473,6 @@ def _check_obligations( return statics, payload, cited -def _first_source(root: Path) -> str: - from .preprocess import preprocess_repository - - _, targets = preprocess_repository(root) - return targets[0].path if targets else "." - - def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None: print(f"manifest={manifest_path}") print(f"target={target}") diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index 8595c28..1141228 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -1168,11 +1168,6 @@ def _mechanisms(case: PairCase) -> tuple[str, ...]: # --- metrics --------------------------------------------------------------- -def _scorable(outcomes: Sequence[PairOutcome]) -> tuple[PairOutcome, ...]: - """Rows that could be scored. An unscorable row leaves every rate rather than counting as a miss (Req 4.1).""" - return tuple(item for item in outcomes if item.unscorable is None) - - def _metrics(outcomes: Sequence[PairOutcome]) -> PairCorpusMetrics: pairs = len(outcomes) detected = sum(1 for item in outcomes if item.detected_vuln) diff --git a/src/openultrasast/provider/openrouter.py b/src/openultrasast/provider/openrouter.py index 3b538e3..915c8d9 100644 --- a/src/openultrasast/provider/openrouter.py +++ b/src/openultrasast/provider/openrouter.py @@ -226,10 +226,3 @@ def _extract_message(payload: object) -> dict[str, object]: if not isinstance(message, dict): raise OpenRouterError("OpenRouter choice did not include a message") return message - - -def _extract_message_content(payload: object) -> str: - content = _extract_message(payload).get("content") - if not isinstance(content, str) or not content.strip(): - raise OpenRouterError("OpenRouter message content was empty") - return content