mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(learn): the experiment_result row keeps one JSON type per column of its object
An experiment's register, run, result and arm_outcome rows share one object, and RustFS's S3 Select infers one schema per object: exp-002's result row carried `units` as an object beside the run row's integer (and `looks` as objects beside the register row's fractions), and after `analyse --record` no row of the experiment could be read (JSONParsingError). The result row now writes `unit_counts` and `looks_taken`; a test asserts one JSON shape per column across the four row kinds. The live object was repaired by dropping the one row (drop_rows) and recording the analysis again. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
8d45483d10
commit
780caaa558
@@ -755,14 +755,18 @@ def analyse(
|
|||||||
|
|
||||||
|
|
||||||
def result_row(manifest: Manifest, pin: str, result: Mapping[str, Any]) -> dict[str, Any]:
|
def result_row(manifest: Manifest, pin: str, result: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
"""The ``experiment_result`` row. It shares one object with the experiment's register, run and outcome rows, and
|
||||||
|
an S3 Select schema is inferred per object, so a column keeps one JSON type across those rows: the unit counts
|
||||||
|
are ``unit_counts`` (the run row's ``units`` is an integer) and the looks ``looks_taken`` (the register row's
|
||||||
|
``looks`` are the fractions)."""
|
||||||
tests = result["tests"]
|
tests = result["tests"]
|
||||||
return {
|
return {
|
||||||
"id": row_id(RESULT_KIND, f"experiment-{manifest.id}", "analyse", manifest.id), "kind": RESULT_KIND,
|
"id": row_id(RESULT_KIND, f"experiment-{manifest.id}", "analyse", manifest.id), "kind": RESULT_KIND,
|
||||||
"repo": experiment_repo(manifest.id), "pin": pin, "run": f"experiment-{manifest.id}", "task": "analyse", "population": POPULATION,
|
"repo": experiment_repo(manifest.id), "pin": pin, "run": f"experiment-{manifest.id}", "task": "analyse", "population": POPULATION,
|
||||||
"split": manifest.id, "image": "host", "experiment": manifest.id, "manifest_digest": manifest.digest,
|
"split": manifest.id, "image": "host", "experiment": manifest.id, "manifest_digest": manifest.digest,
|
||||||
"decision": result["decision"], "reason": result["reason"], "units": result["units"], "spend_usd": result["spend_usd"],
|
"decision": result["decision"], "reason": result["reason"], "unit_counts": result["units"], "spend_usd": result["spend_usd"],
|
||||||
"tests": {n: {"diff": t["estimate"]["diff"], "ci": t["estimate"]["ci"], "ci_vs_zero": t["ci_vs_zero"]} for n, t in tests.items()},
|
"tests": {n: {"diff": t["estimate"]["diff"], "ci": t["estimate"]["ci"], "ci_vs_zero": t["ci_vs_zero"]} for n, t in tests.items()},
|
||||||
"looks": [{"fraction": look["fraction"], "z": look["z"], "crossed": look["crossed"]} for look in result["looks"]],
|
"looks_taken": [{"fraction": look["fraction"], "z": look["z"], "crossed": look["crossed"]} for look in result["looks"]],
|
||||||
} # fmt: skip
|
} # fmt: skip
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -483,3 +483,33 @@ def test_verify_callers_line_switch(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||||||
assert "Known callers: web.py:12 in handler()" in verify.hunt_prompt("app.py", group, "injection", facts)
|
assert "Known callers: web.py:12 in handler()" in verify.hunt_prompt("app.py", group, "injection", facts)
|
||||||
monkeypatch.setenv("OUSAST_VERIFY_CALLERS", "off")
|
monkeypatch.setenv("OUSAST_VERIFY_CALLERS", "off")
|
||||||
assert "Known callers" not in verify.hunt_prompt("app.py", group, "injection", facts)
|
assert "Known callers" not in verify.hunt_prompt("app.py", group, "injection", facts)
|
||||||
|
|
||||||
|
|
||||||
|
def _json_shape(value: object) -> str:
|
||||||
|
"""The JSON type of a value, with an array's element types: what an S3 Select schema inference sees."""
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "array<" + ",".join(sorted({_json_shape(v) for v in value})) + ">"
|
||||||
|
return type(value).__name__
|
||||||
|
|
||||||
|
|
||||||
|
def test_experiment_rows_share_one_json_type_per_column(tmp_path: Path) -> None:
|
||||||
|
"""The register, run and result rows of one experiment land in one object with its outcome rows; RustFS's S3
|
||||||
|
Select infers one schema per object and fails the whole object on a column with two types (ops/ax/README.md).
|
||||||
|
exp-002's result row carried ``units`` as an object beside the run row's integer, and no row of the experiment
|
||||||
|
could be read afterwards (2026-10-02)."""
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
outcomes = crafted(groups=4, b_better=True)
|
||||||
|
report = ex.analyse(manifest, outcomes, resamples=20, provenance={"code_commit": COMMIT})
|
||||||
|
store = FileStore(tmp_path / "memory")
|
||||||
|
registration = ex.register(store, manifest, commit_of=lambda _p: COMMIT, created="2026-10-02")
|
||||||
|
run_row = {
|
||||||
|
**registration, "id": "run-row", "task": ex.RUN_TASK,
|
||||||
|
**ex.RunSummary(manifest.id, "done", 4, {"A": 4, "B": 4}, {"A": 0, "B": 0}, {}, "f" * 64).as_dict(),
|
||||||
|
} # fmt: skip
|
||||||
|
rows = [registration, run_row, ex.result_row(manifest, COMMIT, report), *(ex.outcome_row(manifest.id, COMMIT, o) for o in outcomes)]
|
||||||
|
shapes: dict[str, set[str]] = {}
|
||||||
|
for row in rows:
|
||||||
|
for name, value in row.items():
|
||||||
|
if value is not None:
|
||||||
|
shapes.setdefault(name, set()).add(_json_shape(value))
|
||||||
|
assert {name: sorted(s) for name, s in shapes.items() if len(s) > 1} == {}
|
||||||
|
|||||||
Reference in New Issue
Block a user