mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
feat(hunt): feature table, producers, labels, metric, artifact, the DSPy hunt program, runner and CLI
hunter-learning-loop groups 1-3 (first increment). The feature table is the contract: one row per touched function with diff, source, semantic and taxonomy columns, provenance per producer, labels by the benchmark's rule. Producers read the repository at head only (a planted advisory id and commit text never reach a feature). The hunt program is a DSPy ReAct loop over a signature whose inputs are the unmasked columns in a fixed order; answers outside the closed family list or citing unseen lines score as unknown. The runner writes records the benchmark scorer reads, resumable by change, program id and mask, and refuses the held-out pilot unless the run is a report run. Research extra [hunt] = dspy, pandas; the core install is unchanged. 43 tests on a scripted model, no network. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
e55e44d981
commit
d9d6335ab3
@@ -0,0 +1,49 @@
|
||||
"""Checks for stable run records and budget accounting."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.hunt.artifact import RunArtifact, UsageLedger
|
||||
|
||||
|
||||
def artifact():
|
||||
return RunArtifact(program_id="program", schema_version=1, mask=("source",), data_digests={"train": "abc"},
|
||||
configurations=[{"seed": 0}], usage={"usd": 0.1, "calls": 1, "replay_hits": 0, "wall_seconds": 2},
|
||||
versions={"dspy": "3.4.0", "litellm": "1", "openultrasast": "2.0.1"}, created="2026-10-13T00:00:00Z")
|
||||
|
||||
|
||||
def test_artifact_round_trip_and_tamper(tmp_path):
|
||||
run = artifact()
|
||||
assert run.artifact_id == artifact().artifact_id
|
||||
path = run.write(tmp_path)
|
||||
assert RunArtifact.read(path) == run
|
||||
payload = json.loads(path.read_text())
|
||||
payload["program_id"] = "tampered"
|
||||
path.write_text(json.dumps(payload))
|
||||
with pytest.raises(ValueError, match="id"):
|
||||
RunArtifact.read(path)
|
||||
|
||||
|
||||
def test_ledger_and_budget_prevent_artifact(tmp_path):
|
||||
run = artifact()
|
||||
ledger = UsageLedger(artifact=run)
|
||||
ledger.add({"usd": 0.25, "calls": 1, "wall_seconds": 2})
|
||||
ledger.add({"usd": 0, "replay_hits": 1, "wall_seconds": 1})
|
||||
assert ledger.usage == {"usd": 0.25, "calls": 1, "replay_hits": 1, "wall_seconds": 3}
|
||||
assert ledger.exhausted({"usd": 0.25, "calls": 10})
|
||||
assert ledger.exhausted({"usd": 10, "calls": 1})
|
||||
assert ledger.finish({"usd": 0.25}) is None
|
||||
assert not list(tmp_path.iterdir())
|
||||
result = ledger.finish({"usd": 1, "calls": 2})
|
||||
assert result is run
|
||||
assert result.usage == ledger.usage
|
||||
assert run.finish({"usd": 0.25}) is None
|
||||
assert run.write(tmp_path, budget={"usd": 0.25}) is None
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-1, float("nan"), float("inf")])
|
||||
def test_invalid_usage_rejected(value):
|
||||
with pytest.raises(ValueError):
|
||||
UsageLedger().add({"usd": value})
|
||||
Reference in New Issue
Block a user