feat(memory): every row carries its kind's queryable fields

RustFS's S3 Select infers an object's schema from its first 1000 rows: a where
field absent there is unbound (EvaluatorBindingDoesNotExist), and a column null
in all of them but set later fails the whole object (JSONParsingError, measured:
999 leading nulls answer, 1000 fail). plane/memory.py now declares per kind the
fields a where may name (QUERY_FIELDS: facts candidates_digest, verdict
candidate/final, example profile, plus the ROW_FIELDS every row has) and writes
their union on every row at write time as strings, "" where one does not apply
(null measured to break). rows() refuses an unknown kind, an undeclared where
field and where = null before any read. `ousast plane memory-normalise
[--store URL] [--dry-run]` rewrites pre-rule rows once, idempotently, with
counts. The fake object client mimics both RustFS failures.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-02 08:03:39 +02:00
co-authored by Claude Opus 5.5
parent 30cde99e2d
commit 132d3ef45c
4 changed files with 261 additions and 14 deletions
+26
View File
@@ -287,6 +287,11 @@ def _main(argv: list[str] | None) -> int:
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)")
plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)")
plane_normalise = plane_sub.add_parser(
"memory-normalise", help='rewrite the store\'s rows to the fixed schema (every queryable field, "" where absent); idempotent'
)
plane_normalise.add_argument("--store", help="the store URL (default OUSAST_MEMORY): file:///<path> or minio://<bucket>[/<prefix>]")
plane_normalise.add_argument("--dry-run", action="store_true", help="count what would be rewritten, write nothing")
plane_engine = plane_sub.add_parser(
"alerts-engine", help="a Run's `alerts` for PHP and languages quick mode does not cover, from the engine image on this host"
)
@@ -1384,6 +1389,8 @@ def _plane(args: argparse.Namespace) -> int:
if args.plane_command == "remember":
manifest = args.plane / "runs" / f"{args.run}.yaml"
return _plane_memory("ingest", manifest if manifest.is_file() else None, run=args.run, population=args.population, split=args.split)
if args.plane_command == "memory-normalise":
return _plane_memory_normalise(args)
if args.plane_command == "status":
print(reconciler.status(args.run, units=args.units))
return 0
@@ -1442,6 +1449,25 @@ def _plane_alerts_engine(args: argparse.Namespace) -> int:
return 0
def _plane_memory_normalise(args: argparse.Namespace) -> int:
"""``ousast plane memory-normalise``: one pass over every row object; prints the counts."""
from .plane import memory
try:
store = memory.open_store(args.store)
got = store.normalise(dry_run=args.dry_run)
except (memory.MemoryStoreError, OSError, ValueError) as exc:
print(f"memory-normalise failed: {exc}", file=sys.stderr)
return 1
verb = "would rewrite" if args.dry_run else "rewrote"
print(
f"memory {store.describe()}: {verb} {got['objects_rewritten']}/{got['objects']} objects; "
f'{got["rows_changed"]}/{got["rows"]} rows filled {got["fields_filled"]} absent or null fields with "" '
f"({', '.join(memory.QUERYABLE)})"
)
return 0
def _plane_memory(
step: str, run_manifest: Path | None, *, run: str | None = None, population: str | None = None, split: str | None = None
) -> int: