feat(corpus-seeded-mechanisms): corpus writer and additive mechanism fields

Task 1.2 (Req 1.3, 1.4).

- Mechanism gains origin, review_tier, pairs, shape, guard with defaults;
  rows written before this spec load unchanged.
- MechanismStore.load folds the append-only log by id (latest row wins)
  so a re-seeded shape carries every pair that taught it.
- append_from_pair writes origin=corpus rows with a deterministic id
  derived from the shape key; corpus_mechanisms selects searchable rows.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/semantic/mechanisms.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import math
 import os
@@ -14,6 +15,7 @@ from ..complexity.map import Hotspot
 from ..findings import StaticFinding
 from ..index import CodeChunk, EmbeddingClient, VectorIndex, VectorRecord, write_vector_index
 from .overlay import OverlayRecord
+from .variants import Shape
 
 DEFAULT_MECHANISM_LOG = Path(".openultrasast/mechanisms.jsonl")
 
@@ -29,6 +31,12 @@ class Mechanism:
     what_made_it_exploitable: str
     source_finding_id: str = ""
     source_repo: str = ""
+    # corpus-seeded-mechanisms (additive; old rows load with these defaults)
+    origin: str = "sandbox"  # "sandbox" (proven finding) | "corpus" (derived from a trusted pair)
+    review_tier: str = ""
+    pairs: tuple[str, ...] = ()
+    shape: dict[str, object] | None = None  # Shape.to_dict(); None on sandbox rows (not searchable)
+    guard: str = "none"
 
 
 class MechanismStore:
@@ -39,13 +47,14 @@ class MechanismStore:
     def load(self) -> tuple[Mechanism, ...]:
         if not self.log_path.is_file():
             return ()
-        records: list[Mechanism] = []
+        by_id: dict[str, Mechanism] = {}  # append-only log; the latest row for an id is its current state
         for line in self.log_path.read_text().splitlines():
             if not line.strip():
                 continue
             payload = json.loads(line)
-            records.append(_mechanism_from_payload(payload))
-        return tuple(records)
+            record = _mechanism_from_payload(payload)
+            by_id[record.id] = record
+        return tuple(by_id.values())
 
     def append(self, mechanism: Mechanism) -> None:
         self.log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -80,6 +89,53 @@ def append_mechanism(
     return mechanism
 
 
+def corpus_mechanism_id(shape: Shape) -> str:
+    """Deterministic id from the shape key, so the same shape gets the same record on every machine (lever-addressable)."""
+    return "corpus:" + hashlib.sha1(shape.key().encode()).hexdigest()[:16]
+
+
+def append_from_pair(
+    store: MechanismStore,
+    shape: Shape,
+    *,
+    summary: str,
+    cwe: str,
+    pair: str,
+    provenance: str,
+    tier: str,
+) -> Mechanism:
+    """Second writer (Req 1.3, 1.4): one record per shape, ``origin = "corpus"``, provenance lists every pair that taught it."""
+    mechanism_id = corpus_mechanism_id(shape)
+    existing = next((item for item in store.load() if item.id == mechanism_id), None)
+    pairs = tuple(existing.pairs) if existing is not None else ()
+    if pair not in pairs:
+        pairs = (*pairs, pair)
+    positions = ", ".join(f"arg{p} ({k})" for p, k in zip(shape.source_positions, shape.source_kinds, strict=True))
+    record = Mechanism(
+        id=mechanism_id,
+        summary=summary,
+        cwe=cwe,
+        language=shape.language,
+        tags=(shape.mechanism, f"mechanism:{shape.mechanism}", f"guard:{shape.guard}", f"provenance:{provenance}", f"tier:{tier}"),
+        keywords=(shape.sink_name, shape.guard, *shape.source_kinds),
+        what_made_it_exploitable=f"{positions} reaches {shape.sink_name}/{shape.arity}; the fix added {shape.guard}",
+        source_finding_id="",
+        source_repo="",
+        origin="corpus",
+        review_tier=tier,
+        pairs=pairs,
+        shape=shape.to_dict(),
+        guard=shape.guard,
+    )
+    store.append(record)
+    return record
+
+
+def corpus_mechanisms(records: Sequence[Mechanism]) -> tuple[Mechanism, ...]:
+    """Records that variant search may use: corpus origin with a shape."""
+    return tuple(item for item in records if item.origin == "corpus" and item.shape is not None)
+
+
 def order_promotions(
     hotspots: Sequence[Hotspot],
     records: Sequence[OverlayRecord],
@@ -256,8 +312,10 @@ def _cosine(left: list[float], right: list[float]) -> float:
 def _mechanism_from_payload(payload: dict[str, object]) -> Mechanism:
     tags = payload.get("tags") or []
     keywords = payload.get("keywords") or []
-    if not isinstance(tags, list) or not isinstance(keywords, list):
-        raise ValueError("mechanism tags and keywords must be lists")
+    pairs = payload.get("pairs") or []
+    shape = payload.get("shape")
+    if not isinstance(tags, list) or not isinstance(keywords, list) or not isinstance(pairs, list):
+        raise ValueError("mechanism tags, keywords and pairs must be lists")
     return Mechanism(
         id=str(payload["id"]),
         summary=str(payload.get("summary", "")),
@@ -268,6 +326,11 @@ def _mechanism_from_payload(payload: dict[str, object]) -> Mechanism:
         what_made_it_exploitable=str(payload.get("what_made_it_exploitable", "")),
         source_finding_id=str(payload.get("source_finding_id", "")),
         source_repo=str(payload.get("source_repo", "")),
+        origin=str(payload.get("origin", "sandbox")),
+        review_tier=str(payload.get("review_tier", "")),
+        pairs=tuple(str(item) for item in pairs),
+        shape=Shape.from_dict(shape).to_dict() if isinstance(shape, dict) else None,
+        guard=str(payload.get("guard", "none")),
     )
 
 

tests/test_mechanism_export.py

@@ -0,0 +1,74 @@
+"""corpus-seeded-mechanisms: additive store fields and the corpus writer (task 1.2, Req 1.3/1.4)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from openultrasast.semantic.mechanisms import MechanismStore, append_mechanism
+
+
+def _shape(**overrides: object):
+    from openultrasast.semantic.variants import Shape
+
+    base: dict[str, object] = {
+        "language": "python",
+        "sink_name": "system",
+        "arity": 1,
+        "source_positions": (0,),
+        "source_kinds": ("fact_source",),
+        "guard": "allowlist_test",
+        "mechanism": "source_reaches_sink",
+    }
+    base.update(overrides)
+    return Shape(**base)  # type: ignore[arg-type]
+
+
+def test_old_sandbox_rows_load_with_corpus_defaults(tmp_path: Path) -> None:
+    store = MechanismStore(tmp_path / "mechanisms.jsonl")
+    append_mechanism(store, summary="s", cwe="CWE-78", language="python", tags=("t",), what_made_it_exploitable="x")
+    # a row written before this spec has no origin/tier/pairs/shape/guard fields at all
+    raw = json.loads((tmp_path / "mechanisms.jsonl").read_text().splitlines()[0])
+    for key in ("origin", "review_tier", "pairs", "shape", "guard"):
+        raw.pop(key, None)
+    (tmp_path / "mechanisms.jsonl").write_text(json.dumps(raw) + "\n")
+    (record,) = store.load()
+    assert (
+        record.origin == "sandbox" and record.review_tier == "" and record.pairs == () and record.shape is None and record.guard == "none"
+    )
+
+
+def test_append_from_pair_dedupes_by_shape_and_extends_provenance(tmp_path: Path) -> None:
+    from openultrasast.semantic.mechanisms import append_from_pair, corpus_mechanisms
+
+    store = MechanismStore(tmp_path / "mechanisms.jsonl")
+    first = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p1", provenance="human", tier="seeded")
+    second = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p2", provenance="agent", tier="seeded")
+    assert first.id == second.id
+    assert (
+        second.pairs == ("p1", "p2") and second.origin == "corpus" and second.review_tier == "seeded" and second.guard == "allowlist_test"
+    )
+    third = append_from_pair(
+        store, _shape(guard="none"), summary="request into os.system", cwe="CWE-78", pair="p3", provenance="human", tier="reviewed"
+    )
+    assert third.id != first.id
+    loaded = store.load()
+    assert len(loaded) == 2 and {r.id for r in loaded} == {first.id, third.id}  # the store is append-only, load folds by id
+    corpus = corpus_mechanisms(loaded)
+    assert {r.id for r in corpus} == {first.id, third.id}
+    assert all(r.shape is not None and r.shape["sink_name"] == "system" for r in corpus)
+    assert "source_reaches_sink" in first.tags and "mechanism:source_reaches_sink" in first.tags
+
+
+def test_corpus_rows_never_carry_source_text_and_ids_are_deterministic(tmp_path: Path) -> None:
+    from openultrasast.semantic.mechanisms import append_from_pair
+
+    a = append_from_pair(
+        MechanismStore(tmp_path / "a.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="p", provenance="human", tier="seeded"
+    )
+    b = append_from_pair(
+        MechanismStore(tmp_path / "b.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="q", provenance="human", tier="seeded"
+    )
+    assert a.id == b.id  # identity is the shape key, not a random uuid: the lever can name records across machines
+    text = (tmp_path / "a.jsonl").read_text()
+    assert "app.py" not in text and "cmd" not in text