feat(learn): excerpts and the example store (decision-engine 6.1)

learn/excerpt.py: the labelled function at its pin as numbered lines, bounded
(80 lines / 4,000 chars for candidates, 40 / 2,000 for memory examples),
centred on focus lines; advisory ids anywhere and comments with security
wording (labels.security_reason, multilingual) become <redacted>; owner,
owner/name, path and commit ids are replaced; a delta unit appends the
base->head diff (<= 40 lines). C/C++ and other brace languages without a
repo-facts declaration pattern use a brace matcher (346 c_cpp pair rows
had no excerpt without it).

learn/examples.py + `ousast learn memory build`: label + feature record +
excerpt -> `example` rows (no rationale), excerpts as content-addressed
blobs; refuses origin: user and sources outside sources.toml (exit 2);
conditional labels skipped; exit 2 when >10% of a source's recorded
candidates have no excerpt. A fixed side that was read but no longer
declares the labelled function is counted apart (absent_fixed_side): a
label defect, not an unread clone.

plane/memory.py: `example` in KINDS; put_blob/get_blob/blob_names on the
MemoryStore base, so FileStore and MinioStore share them (tested on both,
MinIO through the fake object client).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/plane/memory.py

@@ -8,6 +8,8 @@ engine's own. Layout, identical for both backends::
     index.jsonl                               one row per ingested (run, task): sha256 of its rows, row count
     facts/<sha256>.json                       facts.json by content (identical facts stored once)
     repos/<host>__<owner>__<name>/<pin>.jsonl the rows of one repository + 40-hex pin, sorted by id
+    <prefix>/<sha256>.<ext>                   blobs (``put_blob``): excerpts/*.txt by content, embeddings/<model>/*.json
+                                              by excerpt sha, responses/*.json by request sha, programs/*.json
 
 ``OUSAST_MEMORY`` selects the backend: ``file:///path`` (:class:`FileStore`, default ``results_root()/memory``) or
 ``minio://<bucket>[/<prefix>]`` (:class:`MinioStore`, the ``minio`` extra). MinIO's endpoint and credentials come
@@ -51,11 +53,16 @@ POPULATION_ANNOTATION = "openultrasast.io/population"
 SPLIT_ANNOTATION = "openultrasast.io/split"
 SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot"  # the loop's guard: {"catalog", "manifest", "populations"}
 # The decision engine's kinds (learned-decision-engine, Data Models): `features` from the `features` task through
-# `remember`; `label`, `decision`, `experiment`, `arm_outcome` and `experiment_result` for the later tasks.
+# `remember`; `label`, `decision`, `experiment`, `arm_outcome` and `experiment_result` for the later tasks; `example`
+# (a labelled case of the local memory: label + feature record + excerpt sha) from `ousast learn memory build`.
 KINDS = (
     "facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome",
-    "features", "label", "decision", "experiment", "arm_outcome", "experiment_result",
+    "features", "label", "decision", "experiment", "arm_outcome", "experiment_result", "example",
 )  # fmt: skip
+# Blobs (`put_blob`): a top-level prefix, optionally one sub-directory (`embeddings/<model-slug>`), and a 64-hex name.
+_BLOB_PREFIX = re.compile(r"[a-z]+(?:/[A-Za-z0-9._-]+)?")
+_BLOB_NAME = re.compile(r"[0-9a-f]{64}")
+BLOB_SUFFIX = {"excerpts": ".txt", "labels": ".jsonl"}  # every other prefix holds JSON
 ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
 TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
 MIN_FREE_BYTES = 1 << 30
@@ -112,6 +119,15 @@ def parse_memory_key(text: str) -> dict[str, str]:
     return {k: str(v) for k, v in key.items()}
 
 
+def blob_key(prefix: str, name: str) -> str:
+    """The key of a blob: ``<prefix>/<name><suffix>``; a prefix or name outside the layout is refused."""
+    if not _BLOB_PREFIX.fullmatch(prefix) or ".." in prefix or prefix.split("/", 1)[0] in ("repos", "facts", "index.jsonl"):
+        raise MemoryStoreError(f"blob prefix {prefix!r} is not a blob prefix of the layout")
+    if not _BLOB_NAME.fullmatch(name):
+        raise MemoryStoreError(f"blob name {name!r} is not a 64-hex sha256")
+    return f"{prefix}/{name}{BLOB_SUFFIX.get(prefix.split('/', 1)[0], '.json')}"
+
+
 def validate_row(row: object, where: str = "row") -> dict[str, Any]:
     if not isinstance(row, dict):
         raise MemoryStoreError(f"{where}: a row must be a JSON object")
@@ -235,6 +251,20 @@ class MemoryStore(ABC):
             return None
         return found[0]
 
+    def get_blob(self, prefix: str, name: str, *, verify: bool = True) -> bytes | None:
+        """The blob ``<prefix>/<name>``. With ``verify`` (a content-addressed blob) one whose content no longer hashes
+        to its name is dropped with a warning and reported absent; keyed blobs (an embedding by its excerpt's sha, a
+        response by its request's sha) pass ``verify=False``."""
+        key = blob_key(prefix, name)
+        found = self._get(key)
+        if found is None:
+            return None
+        if verify and hashlib.sha256(found[0]).hexdigest() != name:
+            log.warning("memory %s: %s no longer matches its sha256; dropped", self.describe(), key)
+            self._delete(key)
+            return None
+        return found[0]
+
     def rows(
         self, repo: str | None = None, pin: str | None = None, kind: str | None = None, where: Mapping[str, Any] | None = None
     ) -> list[Record]:
@@ -260,6 +290,23 @@ class MemoryStore(ABC):
             self._put(key, data)
         return sha
 
+    def put_blob(self, prefix: str, data: bytes, *, name: str | None = None) -> str:
+        """Store ``data`` under ``<prefix>/<sha256(data)>`` (content-addressed; identical data stored once) or under
+        ``name`` when the blob is keyed by something else (the excerpt an embedding is of). A blob already present is
+        not rewritten: a key names one content. Returns the name."""
+        name = name or hashlib.sha256(data).hexdigest()
+        key = blob_key(prefix, name)
+        if self._get(key) is None:
+            self._check_space()
+            self._put(key, data)
+        return name
+
+    def blob_names(self, prefix: str) -> list[str]:
+        """The names of the blobs under ``prefix``, sorted."""
+        blob_key(prefix, "0" * 64)
+        suffix = BLOB_SUFFIX.get(prefix.split("/", 1)[0], ".json")
+        return sorted(k.rsplit("/", 1)[1].removesuffix(suffix) for k in self._keys(prefix + "/") if k.count("/") == prefix.count("/") + 1)
+
     def put_row(self, row: Mapping[str, Any]) -> None:
         self.put_rows([row])