mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(hunt): contamination check (4.5): runs that used a reported repository are marked and kept out of any gain
contamination.py reads the repositories named by the held-out pilot manifest and by every benchmark report (accepted_repos.json and NOTES.md links under benchmarks/measurements), marks each run directory clean or contaminated with the report digest and time, and refuses to count an unchecked run toward a gain. ablation.analyse carries excluded_from_gain when its run is contaminated. Implemented by Codex (gpt-6-astra), gated here; the benchmark import is lazy so the package imports without the benchmarks tree. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
6e5d060de6
commit
dbaaa86d5b
@@ -72,7 +72,7 @@
|
||||
- For every annotation, its effect when removed from the full set and when added to the baseline, with intervals and counts; an annotation stays only on a held-out effect whose interval excludes zero or when a kept annotation needs it; one that raises suspicions without raising recall is dropped and recorded
|
||||
- Done when the table is written for a fixture ablation and the rule's decisions match the fixture's expectations
|
||||
- _Requirements: 4.4, 4.5, 4.6_
|
||||
- [ ] 4.5 Contamination check
|
||||
- [x] 4.5 Contamination check
|
||||
- A training repository later found in the pilot or in a benchmark report marks every run that used it as contaminated and excludes those runs from any gain
|
||||
- Done when a fixture run is marked contaminated after its repository is added to a report list
|
||||
- _Requirements: 2.7_
|
||||
|
||||
@@ -394,10 +394,19 @@ def analyse(out_dir: Path | str, manifest: Manifest, changes: Sequence[Mapping[s
|
||||
for arm, mask in manifest.arms.items()
|
||||
}
|
||||
result["dry_run"] = sum(c["kind"] == "vulnerability" for c in changes) < 40
|
||||
contamination_path = out / "contamination.json"
|
||||
if contamination_path.exists():
|
||||
contamination = json.loads(contamination_path.read_text(encoding="utf-8"))
|
||||
result["contaminated"] = bool(contamination["contaminated"])
|
||||
result["contaminating_repositories"] = contamination["repositories"]
|
||||
if result["contaminated"]:
|
||||
result["excluded_from_gain"] = True
|
||||
_write(out / "analysis.json", result)
|
||||
table = importance(result, annotations=[name for name, _, _ in INPUTS])
|
||||
_write(out / "importance.json", json.loads(table.to_json(orient="records")))
|
||||
decide(table, needs={name: set(deps) for name, deps in manifest.rule.get("needs", {}).items()})
|
||||
decisions = decide(table, needs={name: set(deps) for name, deps in manifest.rule.get("needs", {}).items()})
|
||||
if result.get("excluded_from_gain"):
|
||||
_write(out / "decisions.json", {**decisions, "excluded_from_gain": True})
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Mark runs that used a reported repository and keep them out of gain reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import tomllib
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _eligibility():
|
||||
"""The benchmark's name normaliser and GitHub URL pattern, imported late so the package imports without the benchmarks tree."""
|
||||
from benchmarks.unseen.eligibility import URL, normalize
|
||||
|
||||
return URL, normalize
|
||||
|
||||
|
||||
def report_repositories(root: Path) -> set[str]:
|
||||
"""Read repository identities from the pilot and benchmark reports under root.
|
||||
|
||||
The pilot's repository entries contribute their id, repository and URL fields.
|
||||
Each measurements/*/accepted_repos.json is a list of names. GitHub links in
|
||||
measurements/*/NOTES.md also count. Missing sources contribute nothing, while
|
||||
unreadable or malformed files raise. Advisory and label fields are not inspected.
|
||||
"""
|
||||
URL, normalize = _eligibility()
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(root)
|
||||
names = set()
|
||||
pilot = root / "benchmarks/valid/pilot/pool-valid.toml"
|
||||
if pilot.exists():
|
||||
data = tomllib.loads(pilot.read_text(encoding="utf-8"))
|
||||
for entry in data.get("repository", []):
|
||||
for key in ("id", "repository", "url"):
|
||||
if key in entry:
|
||||
names.add(normalize(entry[key]))
|
||||
measurements = root / "benchmarks/measurements"
|
||||
for path in sorted(measurements.glob("*/accepted_repos.json")):
|
||||
entries = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(entries, list) or any(not isinstance(entry, str) for entry in entries):
|
||||
raise ValueError(f"repository report must be a list of names: {path}")
|
||||
names.update(normalize(entry) for entry in entries)
|
||||
for path in sorted(measurements.glob("*/NOTES.md")):
|
||||
names.update(normalize(match[1].rstrip(".")) for match in URL.finditer(path.read_text(encoding="utf-8")))
|
||||
return names
|
||||
|
||||
|
||||
def _records(run_dir: Path) -> list[dict]:
|
||||
"""Read JSON objects with a change id, skipping artifacts and other metadata."""
|
||||
records = []
|
||||
for path in sorted(run_dir.glob("*.json")):
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(value, dict) and "id" in value:
|
||||
records.append(value)
|
||||
return records
|
||||
|
||||
|
||||
def run_repositories(run_dir: Path) -> set[str]:
|
||||
"""Read study.json changes' repo fields, or resolve each record's repository.
|
||||
|
||||
A record's repo or repository field takes precedence. The current runner writes
|
||||
neither, so its record id is joined to inputs.json item_id and change.repo is
|
||||
read there. Inputs without a record do not count. No advisory or label fields
|
||||
are inspected. Raise if any change lacks a repository or no repositories exist.
|
||||
"""
|
||||
_, normalize = _eligibility()
|
||||
run_dir = Path(run_dir)
|
||||
study = run_dir / "study.json"
|
||||
names = set()
|
||||
if study.exists():
|
||||
for change in json.loads(study.read_text(encoding="utf-8"))["changes"]:
|
||||
if not change.get("repo"):
|
||||
raise ValueError(f"missing repository in study: {run_dir}")
|
||||
names.add(normalize(change["repo"]))
|
||||
else:
|
||||
inputs = None
|
||||
for record in _records(run_dir):
|
||||
repo = record.get("repo") or record.get("repository")
|
||||
if not repo:
|
||||
if inputs is None:
|
||||
path = run_dir / "inputs.json"
|
||||
entries = json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
|
||||
inputs = {entry["item_id"]: entry["change"] for entry in entries}
|
||||
repo = inputs.get(record["id"], {}).get("repo")
|
||||
if not repo:
|
||||
raise ValueError(f"missing repository for record {record['id']} in {run_dir}")
|
||||
names.add(normalize(repo))
|
||||
if not names:
|
||||
raise ValueError(f"no repository named by run: {run_dir}")
|
||||
return names
|
||||
|
||||
|
||||
def check(run_dir: Path, reports: set[str]) -> dict:
|
||||
"""Atomically save the overlap and check time, including when the run is clean.
|
||||
|
||||
The digest is SHA256 of sorted, normalized report names joined by newlines,
|
||||
with no trailing newline. Repository resolution must succeed before writing.
|
||||
"""
|
||||
_, normalize = _eligibility()
|
||||
reports = {normalize(name) for name in reports}
|
||||
overlap = sorted(run_repositories(run_dir) & reports)
|
||||
result = {
|
||||
"contaminated": bool(overlap),
|
||||
"repositories": overlap,
|
||||
"reports_digest": hashlib.sha256("\n".join(sorted(reports)).encode("utf-8")).hexdigest(),
|
||||
"checked_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
path = Path(run_dir) / "contamination.json"
|
||||
temporary = path.with_suffix(path.suffix + ".partial")
|
||||
temporary.write_text(json.dumps(result, sort_keys=True) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
return result
|
||||
|
||||
|
||||
def mark_all(runs_root: Path, reports: set[str]) -> dict[str, dict]:
|
||||
"""Check each immediate child directory containing study.json or change records.
|
||||
|
||||
Return checks keyed by directory name. Unresolvable repositories raise instead
|
||||
of producing a clean check. Directories containing only metadata are skipped.
|
||||
"""
|
||||
results = {}
|
||||
for path in sorted(Path(runs_root).iterdir()):
|
||||
if path.is_dir() and ((path / "study.json").is_file() or _records(path)):
|
||||
results[path.name] = check(path, reports)
|
||||
return results
|
||||
|
||||
|
||||
def clean_runs(run_dirs: Iterable[Path]) -> list[Path]:
|
||||
"""Return checked, uncontaminated runs in input order.
|
||||
|
||||
Raise ValueError listing all unchecked paths if any contamination.json is
|
||||
absent. A missing or non-boolean contaminated value also raises; only an
|
||||
explicit false value qualifies a run for reporting gain.
|
||||
"""
|
||||
run_dirs = [Path(path) for path in run_dirs]
|
||||
unchecked = [str(path) for path in run_dirs if not (path / "contamination.json").is_file()]
|
||||
if unchecked:
|
||||
raise ValueError("unchecked runs: " + ", ".join(unchecked))
|
||||
clean = []
|
||||
for path in run_dirs:
|
||||
result = json.loads((path / "contamination.json").read_text(encoding="utf-8"))
|
||||
if not isinstance(result, dict) or type(result.get("contaminated")) is not bool:
|
||||
raise ValueError(f"invalid contamination check: {path}")
|
||||
if not result["contaminated"]:
|
||||
clean.append(path)
|
||||
return clean
|
||||
@@ -0,0 +1,159 @@
|
||||
"""A report added later must exclude every run that used its repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("dspy")
|
||||
pytest.importorskip("pandas")
|
||||
from test_hunt_ablation import finding, write_records # noqa: E402
|
||||
from test_hunt_ablation import population as population # noqa: E402
|
||||
|
||||
from openultrasast.hunt import ablation as ab # noqa: E402
|
||||
from openultrasast.hunt import contamination as ct # noqa: E402
|
||||
from openultrasast.hunt import runner # noqa: E402
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
|
||||
def test_report_repositories_reads_all_sources_and_normalizes(tmp_path):
|
||||
pilot = tmp_path / "benchmarks/valid/pilot/pool-valid.toml"
|
||||
pilot.parent.mkdir(parents=True)
|
||||
pilot.write_text(
|
||||
'[[repository]]\nid = "Owner/Pilot-ID"\nrepository = "Owner/Pilot-Name.git"\n'
|
||||
'url = "https://github.com/Owner/Pilot-URL.git"\n'
|
||||
'advisory = "https://github.com/ignore/advisory"\nlabel = "https://github.com/ignore/label"\n'
|
||||
)
|
||||
measurements = tmp_path / "benchmarks/measurements"
|
||||
write_json(measurements / "first/accepted_repos.json", ["Owner/Accepted.git", "https://github.com/Owner/Accepted/"])
|
||||
write_json(measurements / "second/accepted_repos.json", ["Other/Accepted"])
|
||||
(measurements / "first/NOTES.md").write_text("[source](https://github.com/Owner/Noted.git/tree/main) and github.com/Other/Plain.\n")
|
||||
(measurements / "second/NOTES.md").write_text("See https://GITHUB.COM/Owner/Second#readme\n")
|
||||
write_json(measurements / "first/labels.json", {"repo": "ignore/labels"})
|
||||
assert ct.report_repositories(tmp_path) == {
|
||||
"owner/pilot-id",
|
||||
"owner/pilot-name",
|
||||
"owner/pilot-url",
|
||||
"owner/accepted",
|
||||
"other/accepted",
|
||||
"owner/noted",
|
||||
"other/plain",
|
||||
"owner/second",
|
||||
}
|
||||
|
||||
|
||||
def record_run(path, repo="Owner/Training.git"):
|
||||
for change_id in ("first", "second"):
|
||||
write_json(runner.record_path(path, change_id), {"id": change_id, "done": True, "push": {}})
|
||||
write_json(
|
||||
path / "inputs.json",
|
||||
[
|
||||
{"item_id": change_id, "change": {"repo": repo, "label": "ignore/label", "advisory": "ignore/advisory"}}
|
||||
for change_id in ("first", "second")
|
||||
]
|
||||
+ [{"item_id": "not-run", "change": {"repo": "unused/input"}}],
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def test_run_becomes_contaminated_after_report_is_added(tmp_path):
|
||||
run_dir = record_run(tmp_path / "run")
|
||||
accepted = tmp_path / "benchmarks/measurements/new/accepted_repos.json"
|
||||
write_json(accepted, ["Other/Repo", "Another/Repo"])
|
||||
assert ct.run_repositories(run_dir) == {"owner/training"}
|
||||
clean = ct.check(run_dir, ct.report_repositories(tmp_path))
|
||||
assert clean["contaminated"] is False and clean["repositories"] == []
|
||||
assert json.loads((run_dir / "contamination.json").read_text()) == clean
|
||||
assert clean["reports_digest"] == hashlib.sha256(b"another/repo\nother/repo").hexdigest()
|
||||
assert datetime.fromisoformat(clean["checked_at"]).tzinfo is not None
|
||||
assert ct.clean_runs([run_dir]) == [run_dir]
|
||||
|
||||
write_json(accepted, ["Other/Repo", "https://github.com/OWNER/Training.git"])
|
||||
contaminated = ct.check(run_dir, ct.report_repositories(tmp_path))
|
||||
assert contaminated["contaminated"] is True
|
||||
assert contaminated["repositories"] == ["owner/training"]
|
||||
assert contaminated["reports_digest"] != clean["reports_digest"]
|
||||
assert json.loads((run_dir / "contamination.json").read_text()) == contaminated
|
||||
assert not (run_dir / "contamination.json.partial").exists()
|
||||
assert ct.clean_runs([run_dir]) == []
|
||||
|
||||
|
||||
def test_run_repositories_uses_direct_fields_and_study_first(tmp_path):
|
||||
write_json(tmp_path / "record.json", {"id": "change", "repo": "Owner/Direct.git"})
|
||||
write_json(tmp_path / "inputs.json", [{"item_id": "change", "change": {"repo": "unused/fallback"}}])
|
||||
assert ct.run_repositories(tmp_path) == {"owner/direct"}
|
||||
write_json(tmp_path / "study.json", {"changes": [{"id": "change", "repo": "https://github.com/Owner/Study.git"}]})
|
||||
assert ct.run_repositories(tmp_path) == {"owner/study"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("contents", [{}, {"id": "unresolved"}])
|
||||
def test_run_repositories_refuses_missing_repository(tmp_path, contents):
|
||||
write_json(tmp_path / "record.json", contents)
|
||||
with pytest.raises(ValueError, match="repository"):
|
||||
ct.check(tmp_path, set())
|
||||
assert not (tmp_path / "contamination.json").exists()
|
||||
|
||||
|
||||
def test_run_repositories_refuses_partly_resolved_run(tmp_path):
|
||||
record_run(tmp_path)
|
||||
write_json(tmp_path / "unresolved.json", {"id": "missing"})
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
ct.run_repositories(tmp_path)
|
||||
|
||||
|
||||
def test_mark_all_checks_records_and_studies_but_not_metadata(tmp_path):
|
||||
record_run(tmp_path / "records")
|
||||
write_json(tmp_path / "study/study.json", {"changes": [{"repo": "OWNER/Training"}]})
|
||||
write_json(tmp_path / "other/study.json", {"changes": [{"repo": "Other/Repo"}]})
|
||||
write_json(tmp_path / "metadata/artifact.json", {"artifact_id": "artifact"})
|
||||
result = ct.mark_all(tmp_path, {"owner/training"})
|
||||
assert set(result) == {"records", "study", "other"}
|
||||
assert result["records"]["contaminated"] is result["study"]["contaminated"] is True
|
||||
assert result["other"]["contaminated"] is False
|
||||
assert not (tmp_path / "metadata/contamination.json").exists()
|
||||
|
||||
|
||||
def test_clean_runs_refuses_all_unchecked_names(tmp_path):
|
||||
clean = record_run(tmp_path / "clean")
|
||||
ct.check(clean, set())
|
||||
with pytest.raises(ValueError, match="unchecked.*first.*second"):
|
||||
ct.clean_runs([clean, tmp_path / "first", tmp_path / "second"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, 0, "false"])
|
||||
def test_clean_runs_refuses_invalid_check(tmp_path, value):
|
||||
write_json(tmp_path / "contamination.json", {"contaminated": value})
|
||||
with pytest.raises(ValueError, match="contamination"):
|
||||
ct.clean_runs([tmp_path])
|
||||
|
||||
|
||||
def test_ablation_preserves_analysis_but_excludes_contaminated_gain(population, tmp_path):
|
||||
_, _, changes, _, original = population
|
||||
manifest = replace(original, sample={**original.sample, "repeats": 1})
|
||||
out = tmp_path / "ablation"
|
||||
write_records(out, manifest, changes, {"full": [[finding()]] * len(changes), "without-roles": [[]] * len(changes)})
|
||||
write_json(out / "study.json", {"manifest_id": ab.manifest_id(manifest), "changes": changes})
|
||||
ct.check(out, set())
|
||||
clean = ab.analyse(out, manifest, changes)
|
||||
assert clean["contaminated"] is False and clean["contaminating_repositories"] == []
|
||||
assert not clean.get("excluded_from_gain", False)
|
||||
importance = json.loads((out / "importance.json").read_text())
|
||||
decisions = json.loads((out / "decisions.json").read_text())
|
||||
|
||||
write_json(tmp_path / "benchmarks/measurements/later/accepted_repos.json", [changes[0]["repo"]])
|
||||
ct.check(out, ct.report_repositories(tmp_path))
|
||||
result = ab.analyse(out, manifest, changes)
|
||||
assert result["contaminated"] is True and result["excluded_from_gain"] is True
|
||||
assert result["contaminating_repositories"] == [changes[0]["repo"]]
|
||||
assert result["heldout"] == clean["heldout"]
|
||||
assert json.loads((out / "analysis.json").read_text())["excluded_from_gain"] is True
|
||||
assert json.loads((out / "importance.json").read_text()) == importance
|
||||
assert json.loads((out / "decisions.json").read_text()) == {**decisions, "excluded_from_gain": True}
|
||||
Reference in New Issue
Block a user