feat(learn): excerpts and the example store (decision-engine 6.1)

learn/excerpt.py: the labelled function at its pin as numbered lines, bounded
(80 lines / 4,000 chars for candidates, 40 / 2,000 for memory examples),
centred on focus lines; advisory ids anywhere and comments with security
wording (labels.security_reason, multilingual) become <redacted>; owner,
owner/name, path and commit ids are replaced; a delta unit appends the
base->head diff (<= 40 lines). C/C++ and other brace languages without a
repo-facts declaration pattern use a brace matcher (346 c_cpp pair rows
had no excerpt without it).

learn/examples.py + `ousast learn memory build`: label + feature record +
excerpt -> `example` rows (no rationale), excerpts as content-addressed
blobs; refuses origin: user and sources outside sources.toml (exit 2);
conditional labels skipped; exit 2 when >10% of a source's recorded
candidates have no excerpt. A fixed side that was read but no longer
declares the labelled function is counted apart (absent_fixed_side): a
label defect, not an unread clone.

plane/memory.py: `example` in KINDS; put_blob/get_blob/blob_names on the
MemoryStore base, so FileStore and MinioStore share them (tested on both,
MinIO through the fake object client).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_learn_excerpt.py

@@ -0,0 +1,138 @@
+"""Excerpts for the decision engine (learned-decision-engine design 4.2): bounds, centring, no identities, advisory ids
+and security wording in comments redacted, the delta diff."""
+
+from __future__ import annotations
+
+from openultrasast.learn.excerpt import (
+    CANDIDATE,
+    EXAMPLE,
+    REDACTED,
+    Bounds,
+    delta_diff,
+    excerpt,
+    excerpt_sha,
+    identities_of,
+    normalise,
+    redact,
+)
+
+PY = [
+    "import os",
+    "",
+    "def handler(request):",
+    "    # CVE-2021-1234: prevent SQL injection by quoting",
+    "    name = request.args['name']",
+    '    """Fetch the user. Fixes GHSA-abcd-efgh-ijkm."""',
+    "    query = 'SELECT * FROM users WHERE name = ' + name  # build the query",
+    "    return db.execute(query)",
+    "",
+    "def other():",
+    "    return 1",
+]
+
+
+def _long_function(n: int, width: int = 10) -> list[str]:
+    return ["def big(x):"] + [f"    v{i} = x + {i}" + " " * width for i in range(n)] + ["    return v0", "", "def after():", "    pass"]
+
+
+def test_function_only_numbered_from_its_first_line() -> None:
+    shown = excerpt(PY, "python", "handler")
+    assert shown is not None
+    lines = shown.text.splitlines()
+    assert lines[0].split()[0] == "3" and "def handler" in lines[0]
+    assert "def other" not in shown.text and shown.first == 3 and shown.last == 9
+    assert shown.sha == excerpt_sha(shown.text) and not shown.truncated
+
+
+def test_unread_file_or_unknown_function_is_none_never_empty() -> None:
+    assert excerpt(None, "python", "handler") is None
+    assert excerpt([], "python", "handler") is None
+    assert excerpt(PY, "python", "missing") is None
+    assert excerpt(PY, "cobol", "handler") is None
+
+
+def test_security_comments_and_advisory_ids_are_redacted() -> None:
+    shown = excerpt(PY, "python", "handler")
+    assert shown is not None
+    assert "CVE-2021" not in shown.text and "GHSA" not in shown.text and "injection" not in shown.text
+    assert "prevent" not in shown.text and REDACTED in shown.text
+    assert "# build the query" in shown.text  # an ordinary comment stays
+    assert "query = 'SELECT * FROM users WHERE name = ' + name" in shown.text  # code is never redacted
+
+
+def test_redaction_covers_block_comments_and_other_languages() -> None:
+    php = ["function f($id) {", "  /* sanitize to avoid XSS", "     (see report) */", "  echo $id; // 修复越权", "}"]
+    out = redact(php, "php")
+    assert "XSS" not in "\n".join(out) and "修复越权" not in out[3] and out[3].startswith("  echo $id;")
+    assert "#" not in redact(["x = '#not a comment'"], "python")[0].replace("'#not a comment'", "")
+    assert redact(["s = 'CVE-2020-0001'"], "python") == [f"s = '{REDACTED}'"]
+
+
+def test_no_path_repository_or_commit_in_the_excerpt() -> None:
+    lines = [
+        "def handler(request):",
+        "    # see https://github.com/acme-corp/webapp/blob/0123456789abcdef0123456789abcdef01234567/app/views.py",
+        "    log('app/views.py')",
+        "    return request",
+    ]
+    ids = identities_of("https://github.com/acme-corp/webapp", "app/views.py", ["0123456789abcdef0123456789abcdef01234567"])
+    shown = excerpt(lines, "python", "handler", identities=ids)
+    assert shown is not None
+    for identity in ("acme-corp", "0123456789abcdef", "app/views.py"):
+        assert identity not in shown.text
+    assert "webapp" in shown.text or REDACTED in shown.text  # the bare name is code-level, the owner and slug are not
+
+
+def test_bounds_lines_and_characters() -> None:
+    lines = _long_function(200)
+    shown = excerpt(lines, "python", "big", bounds=CANDIDATE)
+    assert shown is not None and shown.truncated
+    assert len(shown.text.splitlines()) <= CANDIDATE.lines and len(shown.text) <= CANDIDATE.chars
+    small = excerpt(lines, "python", "big", bounds=EXAMPLE)
+    assert small is not None and len(small.text.splitlines()) <= EXAMPLE.lines and len(small.text) <= EXAMPLE.chars
+    wide = _long_function(60, width=120)
+    tight = excerpt(wide, "python", "big", bounds=Bounds(80, 2000))
+    assert tight is not None and len(tight.text) <= 2000 and tight.truncated
+
+
+def test_window_centres_on_the_focus_lines() -> None:
+    lines = _long_function(300)
+    shown = excerpt(lines, "python", "big", bounds=Bounds(40, 100_000), focus=[200])
+    assert shown is not None and shown.first <= 200 <= shown.last
+    assert abs((shown.first + shown.last) / 2 - 200) <= 1
+    chars = excerpt(lines, "python", "big", bounds=Bounds(80, 1200), focus=[200])
+    assert chars is not None and chars.first <= 200 <= chars.last
+
+
+def test_delta_diff_is_redacted_and_bounded() -> None:
+    base = ["def f(x):", "    return escape(x)"]
+    head = ["def f(x):", "    # CVE-2022-2222 sanitize later", "    return x"]
+    diff = delta_diff(base, head, "python")
+    assert diff.startswith("@@") and "+    return x" in diff and "-    return escape(x)" in diff
+    assert "CVE-2022" not in diff and "sanitize" not in diff and "---" not in diff
+    long = delta_diff([f"a{i}" for i in range(100)], [f"b{i}" for i in range(100)], "python", max_lines=40)
+    assert len(long.splitlines()) == 40 and long.splitlines()[-1].startswith("...")
+    shown = excerpt(head, "python", "f", diff=diff)
+    assert shown is not None and "\ndiff:\n@@" in shown.text
+
+
+def test_normalised_text_names_the_content() -> None:
+    assert normalise("a  \r\nb\n\n\n") == "a\nb\n"
+    assert excerpt_sha("a  \nb") == excerpt_sha("a\nb\n")
+
+
+def test_brace_languages_without_a_declaration_pattern() -> None:
+    c = [
+        "/* CVE-2016-0001: heap overflow fixed */",
+        "static int",
+        "parse_header(const char *buf, size_t len)",
+        "{",
+        "    if (len > 4) { memcpy(out, buf, len); }",
+        "    return parse_header_inner(buf); /* not a definition */",
+        "}",
+        "int other(void) { return parse_header(0, 0); }",
+    ]
+    shown = excerpt(c, "c_cpp", "parse_header")
+    assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text
+    assert excerpt(c, "cpp", "Parser::parse_header") is not None
+    assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None  # a prototype is not a body