feat(learn): excerpts and the example store (decision-engine 6.1)

learn/excerpt.py: the labelled function at its pin as numbered lines, bounded
(80 lines / 4,000 chars for candidates, 40 / 2,000 for memory examples),
centred on focus lines; advisory ids anywhere and comments with security
wording (labels.security_reason, multilingual) become <redacted>; owner,
owner/name, path and commit ids are replaced; a delta unit appends the
base->head diff (<= 40 lines). C/C++ and other brace languages without a
repo-facts declaration pattern use a brace matcher (346 c_cpp pair rows
had no excerpt without it).

learn/examples.py + `ousast learn memory build`: label + feature record +
excerpt -> `example` rows (no rationale), excerpts as content-addressed
blobs; refuses origin: user and sources outside sources.toml (exit 2);
conditional labels skipped; exit 2 when >10% of a source's recorded
candidates have no excerpt. A fixed side that was read but no longer
declares the labelled function is counted apart (absent_fixed_side): a
label defect, not an unread clone.

plane/memory.py: `example` in KINDS; put_blob/get_blob/blob_names on the
MemoryStore base, so FileStore and MinioStore share them (tested on both,
MinIO through the fake object client).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_learn_examples.py

@@ -0,0 +1,157 @@
+"""The example store (learned-decision-engine design 4.2, task 6.1): label + feature record + excerpt -> ``example``
+rows, the refusals, the 10% unread-excerpt failure, and content-addressed blobs on both backends."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import pytest
+from test_plane_memory import FakeObjects
+
+from openultrasast.cli import main
+from openultrasast.learn.examples import (
+    ExampleBuildError,
+    build_examples,
+    label_pin,
+    load_examples,
+    record_key,
+    records_by_key,
+)
+from openultrasast.learn.features import Part, quick_part, record
+from openultrasast.learn.labels import Source, Sources
+from openultrasast.plane.memory import FileStore, MemoryStore, MemoryStoreError, MinioStore
+
+PIN = "0123456789abcdef0123456789abcdef01234567"
+SOURCES = Sources(
+    (Source("pairs", "pairs", (), (), "2026-09-30", "development"), Source("population-v1", "population", (), (), "2026-09-25", "spent")),
+    frozenset(),
+    {},
+    Path("sources.toml"),
+)
+CODE = {
+    "vulnerable": ["def handler(request):", "    # CVE-2021-1234 fixed later", "    return run(request.args['q'])", ""],
+    "fixed": ["def handler(request):", "    return run(quote(request.args['q']))", ""],
+}
+
+
+@pytest.fixture(params=["file", "fake-minio"])
+def store(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[MemoryStore]:
+    yield FileStore(tmp_path / "memory") if request.param == "file" else MinioStore(FakeObjects("works"), "bucket", "p")
+
+
+def label(**kw: Any) -> dict[str, Any]:
+    base = {
+        "candidate": "app.py::handler", "family": "injection", "label": 1, "source": "population-v1", "source_ref": "case-1",
+        "unit": "pin", "weight": 1.0, "split": "population-v1", "group": "acme/webapp", "repo": "https://github.com/acme/webapp",
+        "language": "python", "created": "2026-09-30", "pin": PIN, "pin_role": "vulnerable", "frameworks": ["flask"],
+        "conditional": None, "direction": None,
+    }  # fmt: skip
+    return {**base, **kw}
+
+
+def feature(lab: Mapping[str, Any], unit: str = "pin") -> dict[str, Any]:
+    parts = {"quick": quick_part([("r1", "enabled")], {})}
+    if unit == "delta":
+        parts["delta"] = Part("ran", {"delta.novelty": "new", "delta.changed_lines": 1, "delta.sanitizer_removed": True})
+    out = record(str(lab["candidate"]), str(lab["family"]), "python", "static", parts, unit=unit)
+    return {**out, "repo": lab["repo"], "pin": label_pin(lab)}
+
+
+def lines_of(lab: Mapping[str, Any], role: str) -> Sequence[str] | None:
+    return CODE[role]
+
+
+def test_example_rows_join_label_record_and_excerpt(store: MemoryStore) -> None:
+    pos, neg = label(), label(label=0, pin_role="fixed", pin="f" * 40)
+    records = records_by_key([feature(pos), feature(neg)])
+    build = build_examples([pos, neg], records, lines_of, sources=SOURCES, store=store)
+    assert build.summary()["examples"] == 2 and not build.failing()
+    stored = load_examples(store)
+    assert sorted(e.label for e in stored) == [0, 1] and {e.group for e in stored} == {"acme/webapp"}
+    row = next(r.row for r in store.rows(kind="example") if r.row["label"] == 1)
+    assert row["kind"] == "example" and row["frameworks"] == ["flask"] and "rationale" not in row
+    text = store.get_blob("excerpts", row["excerpt_sha"])
+    assert text is not None and hashlib.sha256(text).hexdigest() == row["excerpt_sha"]
+    assert b"CVE-2021" not in text
+    assert "acme" not in text.decode() and "app.py" not in text.decode()
+
+
+def test_delta_units_carry_the_diff(store: MemoryStore) -> None:
+    lab = label(unit="delta", direction="introduce")
+    build = build_examples([lab], records_by_key([feature(lab, "delta")]), lines_of, sources=SOURCES, store=store)
+    row = build.rows[0]
+    text = store.get_blob("excerpts", row["excerpt_sha"])
+    assert text is not None and b"\ndiff:\n@@" in text and b"+    return run(request.args['q'])" in text
+
+
+def test_user_origin_and_unlisted_sources_are_refused() -> None:
+    with pytest.raises(ExampleBuildError, match="origin: user"):
+        build_examples([label(origin="user")], {}, lines_of, sources=SOURCES)
+    with pytest.raises(ExampleBuildError, match="not in"):
+        build_examples([label(source="population-v3")], {}, lines_of, sources=SOURCES)
+
+
+def test_conditional_and_unrecorded_labels_never_become_examples() -> None:
+    rows = [label(conditional="signal_at_tip_not_base", label=0), label(source="pairs", pin="", source_ref="p1")]
+    build = build_examples(rows, {}, lines_of, sources=SOURCES)
+    assert build.rows == [] and build.skipped == {"conditional label (condition not evaluated)": 1}
+    assert build.sources["pairs"].no_record == 1
+
+
+def test_more_than_ten_percent_without_excerpt_fails_the_source() -> None:
+    labs = [label(candidate=f"app.py::f{i}", source_ref=f"c{i}") for i in range(10)] + [label()]
+    records = records_by_key([feature(lab) for lab in labs])
+    build = build_examples(labs, records, lines_of, sources=SOURCES)  # f0..f9 are not declared in CODE: no excerpt
+    assert build.sources["population-v1"].no_excerpt == 10 and build.failing() == ["population-v1"]
+    ok = build_examples([label()], records, lines_of, sources=SOURCES)
+    assert ok.failing() == []
+
+
+def test_pair_pins_are_stable_and_distinct_per_side() -> None:
+    vuln, fixed = label(source="pairs", pin="", source_ref="p1"), label(source="pairs", pin="", source_ref="p1", pin_role="fixed")
+    assert label_pin(vuln) != label_pin(fixed) and len(label_pin(vuln)) == 40 and label_pin(vuln) == label_pin(dict(vuln))
+    assert label_pin(label()) == PIN
+    key = record_key(vuln["repo"], label_pin(vuln), "app.py::handler", "injection", "pin")
+    assert key[0] == "github.com/acme/webapp"
+
+
+def test_blobs_are_content_addressed_on_both_backends(store: MemoryStore) -> None:
+    sha = store.put_blob("excerpts", b"x = 1\n")
+    assert sha == hashlib.sha256(b"x = 1\n").hexdigest() and store.get_blob("excerpts", sha) == b"x = 1\n"
+    assert store.put_blob("excerpts", b"x = 1\n") == sha and store.blob_names("excerpts") == [sha]
+    keyed = store.put_blob("embeddings/openai__text-embedding-3-small", b'{"v": 1}', name=sha)
+    assert keyed == sha and store.get_blob("embeddings/openai__text-embedding-3-small", sha, verify=False) == b'{"v": 1}'
+    assert store.get_blob("excerpts", "0" * 64) is None
+    for bad in ("repos", "../x", "facts"):
+        with pytest.raises(MemoryStoreError):
+            store.put_blob(bad, b"x")
+    with pytest.raises(MemoryStoreError):
+        store.get_blob("excerpts", "not-a-sha")
+
+
+def test_a_changed_blob_is_dropped_not_served(tmp_path: Path) -> None:
+    store = FileStore(tmp_path / "m")
+    sha = store.put_blob("excerpts", b"a\n")
+    (tmp_path / "m" / "excerpts" / f"{sha}.txt").write_bytes(b"tampered\n")
+    assert store.get_blob("excerpts", sha) is None and store.blob_names("excerpts") == []
+
+
+def test_cli_build_writes_examples_and_exits_2_on_refusal(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    labels = tmp_path / "labels.jsonl"
+    labels.write_text(json.dumps(label(origin="user")) + "\n")
+    assert main(["learn", "memory", "build", "--labels", str(labels), "--dry-run"]) == 2
+    assert "origin: user" in capsys.readouterr().err
+
+
+def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side() -> None:
+    fixed = label(label=0, pin_role="fixed", pin="f" * 40)
+    moved = {"vulnerable": CODE["vulnerable"], "fixed": ["def guard(request):", "    return True"]}
+    build = build_examples([fixed], records_by_key([feature(fixed)]), lambda lab, role: moved[role], sources=SOURCES)
+    count = build.sources["population-v1"]
+    assert count.absent_fixed_side == 1 and count.no_excerpt == 0 and build.failing() == []
+    unread = build_examples([fixed], records_by_key([feature(fixed)]), lambda lab, role: None, sources=SOURCES)
+    assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"]

tests/test_learn_excerpt.py

@@ -0,0 +1,138 @@
+"""Excerpts for the decision engine (learned-decision-engine design 4.2): bounds, centring, no identities, advisory ids
+and security wording in comments redacted, the delta diff."""
+
+from __future__ import annotations
+
+from openultrasast.learn.excerpt import (
+    CANDIDATE,
+    EXAMPLE,
+    REDACTED,
+    Bounds,
+    delta_diff,
+    excerpt,
+    excerpt_sha,
+    identities_of,
+    normalise,
+    redact,
+)
+
+PY = [
+    "import os",
+    "",
+    "def handler(request):",
+    "    # CVE-2021-1234: prevent SQL injection by quoting",
+    "    name = request.args['name']",
+    '    """Fetch the user. Fixes GHSA-abcd-efgh-ijkm."""',
+    "    query = 'SELECT * FROM users WHERE name = ' + name  # build the query",
+    "    return db.execute(query)",
+    "",
+    "def other():",
+    "    return 1",
+]
+
+
+def _long_function(n: int, width: int = 10) -> list[str]:
+    return ["def big(x):"] + [f"    v{i} = x + {i}" + " " * width for i in range(n)] + ["    return v0", "", "def after():", "    pass"]
+
+
+def test_function_only_numbered_from_its_first_line() -> None:
+    shown = excerpt(PY, "python", "handler")
+    assert shown is not None
+    lines = shown.text.splitlines()
+    assert lines[0].split()[0] == "3" and "def handler" in lines[0]
+    assert "def other" not in shown.text and shown.first == 3 and shown.last == 9
+    assert shown.sha == excerpt_sha(shown.text) and not shown.truncated
+
+
+def test_unread_file_or_unknown_function_is_none_never_empty() -> None:
+    assert excerpt(None, "python", "handler") is None
+    assert excerpt([], "python", "handler") is None
+    assert excerpt(PY, "python", "missing") is None
+    assert excerpt(PY, "cobol", "handler") is None
+
+
+def test_security_comments_and_advisory_ids_are_redacted() -> None:
+    shown = excerpt(PY, "python", "handler")
+    assert shown is not None
+    assert "CVE-2021" not in shown.text and "GHSA" not in shown.text and "injection" not in shown.text
+    assert "prevent" not in shown.text and REDACTED in shown.text
+    assert "# build the query" in shown.text  # an ordinary comment stays
+    assert "query = 'SELECT * FROM users WHERE name = ' + name" in shown.text  # code is never redacted
+
+
+def test_redaction_covers_block_comments_and_other_languages() -> None:
+    php = ["function f($id) {", "  /* sanitize to avoid XSS", "     (see report) */", "  echo $id; // 修复越权", "}"]
+    out = redact(php, "php")
+    assert "XSS" not in "\n".join(out) and "修复越权" not in out[3] and out[3].startswith("  echo $id;")
+    assert "#" not in redact(["x = '#not a comment'"], "python")[0].replace("'#not a comment'", "")
+    assert redact(["s = 'CVE-2020-0001'"], "python") == [f"s = '{REDACTED}'"]
+
+
+def test_no_path_repository_or_commit_in_the_excerpt() -> None:
+    lines = [
+        "def handler(request):",
+        "    # see https://github.com/acme-corp/webapp/blob/0123456789abcdef0123456789abcdef01234567/app/views.py",
+        "    log('app/views.py')",
+        "    return request",
+    ]
+    ids = identities_of("https://github.com/acme-corp/webapp", "app/views.py", ["0123456789abcdef0123456789abcdef01234567"])
+    shown = excerpt(lines, "python", "handler", identities=ids)
+    assert shown is not None
+    for identity in ("acme-corp", "0123456789abcdef", "app/views.py"):
+        assert identity not in shown.text
+    assert "webapp" in shown.text or REDACTED in shown.text  # the bare name is code-level, the owner and slug are not
+
+
+def test_bounds_lines_and_characters() -> None:
+    lines = _long_function(200)
+    shown = excerpt(lines, "python", "big", bounds=CANDIDATE)
+    assert shown is not None and shown.truncated
+    assert len(shown.text.splitlines()) <= CANDIDATE.lines and len(shown.text) <= CANDIDATE.chars
+    small = excerpt(lines, "python", "big", bounds=EXAMPLE)
+    assert small is not None and len(small.text.splitlines()) <= EXAMPLE.lines and len(small.text) <= EXAMPLE.chars
+    wide = _long_function(60, width=120)
+    tight = excerpt(wide, "python", "big", bounds=Bounds(80, 2000))
+    assert tight is not None and len(tight.text) <= 2000 and tight.truncated
+
+
+def test_window_centres_on_the_focus_lines() -> None:
+    lines = _long_function(300)
+    shown = excerpt(lines, "python", "big", bounds=Bounds(40, 100_000), focus=[200])
+    assert shown is not None and shown.first <= 200 <= shown.last
+    assert abs((shown.first + shown.last) / 2 - 200) <= 1
+    chars = excerpt(lines, "python", "big", bounds=Bounds(80, 1200), focus=[200])
+    assert chars is not None and chars.first <= 200 <= chars.last
+
+
+def test_delta_diff_is_redacted_and_bounded() -> None:
+    base = ["def f(x):", "    return escape(x)"]
+    head = ["def f(x):", "    # CVE-2022-2222 sanitize later", "    return x"]
+    diff = delta_diff(base, head, "python")
+    assert diff.startswith("@@") and "+    return x" in diff and "-    return escape(x)" in diff
+    assert "CVE-2022" not in diff and "sanitize" not in diff and "---" not in diff
+    long = delta_diff([f"a{i}" for i in range(100)], [f"b{i}" for i in range(100)], "python", max_lines=40)
+    assert len(long.splitlines()) == 40 and long.splitlines()[-1].startswith("...")
+    shown = excerpt(head, "python", "f", diff=diff)
+    assert shown is not None and "\ndiff:\n@@" in shown.text
+
+
+def test_normalised_text_names_the_content() -> None:
+    assert normalise("a  \r\nb\n\n\n") == "a\nb\n"
+    assert excerpt_sha("a  \nb") == excerpt_sha("a\nb\n")
+
+
+def test_brace_languages_without_a_declaration_pattern() -> None:
+    c = [
+        "/* CVE-2016-0001: heap overflow fixed */",
+        "static int",
+        "parse_header(const char *buf, size_t len)",
+        "{",
+        "    if (len > 4) { memcpy(out, buf, len); }",
+        "    return parse_header_inner(buf); /* not a definition */",
+        "}",
+        "int other(void) { return parse_header(0, 0); }",
+    ]
+    shown = excerpt(c, "c_cpp", "parse_header")
+    assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text
+    assert excerpt(c, "cpp", "Parser::parse_header") is not None
+    assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None  # a prototype is not a body