diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 87dd90c..36608e6 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -54,6 +54,7 @@ from .pairs import ( ) from .policy import assert_rules_resolve, load_policy from .preprocess import preprocess_repository, write_preprocess_artifact +from .provenance import fingerprint from .provider.openrouter import OpenRouterEmbeddingClient, OpenRouterError from .rank import rank_targets, write_rankings from .regress import TRIGGERABLE, CandidateVerdict, run_regression, write_verdicts @@ -223,6 +224,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut write_entry_points(entry_points, run.root / "mapping" / "entry_points.json") targets = attach_reachability_hints(targets, entry_points) write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json") + provenance = runtime.run_stage("provenance", lambda: fingerprint(run.target)) rankings = runtime.run_stage("rank", lambda: rank_targets(targets)) prior_learnings = load_false_positive_learnings(ledger_path) rankings, calibrations = runtime.run_stage("calibrate", lambda: calibrate_rankings(rankings, prior_learnings)) @@ -520,6 +522,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut stages=stages_payload(plan), complexity=complexity_payload, worth_fixing=worth_fixing_payload, + provenance=provenance.to_dict(), ), ) runtime.finish(status="succeeded") diff --git a/src/openultrasast/provenance.py b/src/openultrasast/provenance.py new file mode 100644 index 0000000..7555711 --- /dev/null +++ b/src/openultrasast/provenance.py @@ -0,0 +1,117 @@ +"""Deterministic provenance fingerprint: does this tree look agent-authored? + +pair-corpus-honesty Requirement 6. Signals are files and directories that agent +tooling leaves behind, generation markers in text, and git co-author trailers +over a bounded commit sample. No model is consulted. ``synthetic`` is never +produced by a scan; it is a catalog declaration only. +""" + +from __future__ import annotations + +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path + +PROVENANCE_VALUES = ("human", "agent", "mixed", "synthetic") +AGENT_SHARE_THRESHOLD = 0.85 # VibeApps method: >=85% of sampled commits carry an agent signal +DEFAULT_SAMPLE_COMMITS = 200 + +_AGENT_PATHS = ( + ".claude", + "CLAUDE.md", + "AGENTS.md", + ".cursor", + ".cursorrules", + ".windsurf", + ".windsurfrules", + ".github/copilot-instructions.md", + ".codex", + ".jules", + ".kiro", + ".opencode", +) +_MARKERS = ( + ("Generated with Claude Code", "marker:claude-code"), + (' dict[str, object]: + return { + "value": self.value, + "signals": list(self.signals), + "sampled_commits": self.sampled_commits, + "agent_commits": self.agent_commits, + } + + +def fingerprint(root: Path, *, git: bool = True, sample_commits: int = DEFAULT_SAMPLE_COMMITS) -> Provenance: + """Classify ``root`` from deterministic signals only. Never raises into a scan.""" + signals: list[str] = [] + for relative in _AGENT_PATHS: + if (root / relative).exists(): + signals.append(f"path:{relative}") + for name in _MARKER_FILES: + path = root / name + if not path.is_file(): + continue + try: + text = path.read_text(errors="ignore")[:200_000] + except OSError: + continue + for needle, label in _MARKERS: + if needle in text and label not in signals: + signals.append(label) + sampled = agent = 0 + if git: + sampled, agent, git_signal = _git_trailers(root, sample_commits) + if git_signal: + signals.append(git_signal) + if sampled and agent / sampled >= AGENT_SHARE_THRESHOLD: + value = "agent" + elif any(not signal.startswith("git:") for signal in signals) or agent: + value = "mixed" + else: + value = "human" + return Provenance(value=value, signals=tuple(signals), sampled_commits=sampled, agent_commits=agent) + + +def _git_trailers(root: Path, sample_commits: int) -> tuple[int, int, str]: + """(sampled, agent-signalled, signal-label). Missing git is a signal, not an error.""" + if sample_commits <= 0: + return 0, 0, "" + try: + completed = subprocess.run( + ["git", "-C", str(root), "log", f"-{sample_commits}", "--format=%H%x00%B%x1e"], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return 0, 0, "git:unavailable" + if completed.returncode != 0: + return 0, 0, "git:unavailable" + entries = [entry for entry in completed.stdout.split("\x1e") if entry.strip()] + sampled = len(entries) + agent = sum(1 for entry in entries if _TRAILER_RE.search(entry) or _GENERATED_RE.search(entry)) + if not sampled: + return 0, 0, "git:no-commits" + return sampled, agent, f"git:trailers {agent}/{sampled}" diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index f14bdd4..fdea267 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -173,6 +173,7 @@ def write_manifest( stages: dict[str, object] | None = None, complexity: dict[str, object] | None = None, worth_fixing: dict[str, object] | None = None, + provenance: dict[str, object] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) @@ -207,6 +208,8 @@ def write_manifest( payload["complexity"] = complexity if worth_fixing is not None: payload["worth_fixing"] = worth_fixing + if provenance is not None: + payload["provenance"] = provenance path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000..475b65b --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,57 @@ +"""Deterministic provenance fingerprint (pair-corpus-honesty Req 6).""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from openultrasast.provenance import AGENT_SHARE_THRESHOLD, fingerprint + + +def test_empty_tree_is_human_with_no_signals(tmp_path: Path) -> None: + result = fingerprint(tmp_path, git=False) + assert result.value == "human" + assert result.signals == () + + +def test_agent_config_file_makes_tree_mixed(tmp_path: Path) -> None: + (tmp_path / "AGENTS.md").write_text("# agents\n") + (tmp_path / ".claude").mkdir() + result = fingerprint(tmp_path, git=False) + assert result.value == "mixed" + assert "path:AGENTS.md" in result.signals and "path:.claude" in result.signals + + +def test_lovable_marker_in_index_html(tmp_path: Path) -> None: + (tmp_path / "index.html").write_text('
') + result = fingerprint(tmp_path, git=False) + assert result.value == "mixed" + assert "marker:lovable" in result.signals + + +def test_git_trailer_share_over_threshold_is_agent(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + env = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@x"} + for index in range(10): + (tmp_path / f"f{index}.py").write_text("x = 1\n") + subprocess.run(["git", "-C", str(tmp_path), "add", "."], check=True) + trailer = "\n\nCo-Authored-By: Claude