feat(scan): add initial SAST pipeline

tests/test_calibration.py

@@ -0,0 +1,115 @@
+from pathlib import Path
+
+from openultrasast.calibration import (
+    FalsePositiveReason,
+    FindingOutcome,
+    apply_ranking_calibration,
+    calibrate_ranking,
+    calibrate_rankings,
+    compute_ranking_metrics,
+    learning_from_finding_outcome,
+    record_false_positive_learning,
+    write_false_positive_learnings,
+)
+from openultrasast.findings import StaticFinding
+from openultrasast.rank import RankingScore
+
+
+def _finding(path: str = "auth/login.py") -> StaticFinding:
+    return StaticFinding(
+        finding_id="python-unsafe-eval:auth/login.py:1",
+        path=path,
+        title="Dynamic Python execution needs review",
+        severity="high",
+        confidence="medium",
+        evidence_level="static_corroboration",
+        rationale="pattern matched",
+        line=1,
+        tags=["syscall_entry"],
+        ranking_priority=4.0,
+    )
+
+
+def _ranking(path: str) -> RankingScore:
+    return RankingScore(
+        path=path,
+        surface=4,
+        influence=2,
+        reachability=4,
+        priority=3.6,
+        rationale="heuristic",
+        model_id=None,
+        static_boosts=["syscall_entry"],
+    )
+
+
+def test_false_positive_learning_records_reason_and_scope(tmp_path: Path) -> None:
+    learning = record_false_positive_learning(
+        _finding(),
+        reason=FalsePositiveReason.MISSING_ATTACKER_CONTROL,
+        evidence="Verifier showed input is constant and not attacker-controlled.",
+    )
+    output = tmp_path / "false-positive-learnings.json"
+
+    write_false_positive_learnings([learning], output)
+
+    assert learning.vulnerability_class == "syscall_entry"
+    assert learning.scope == "auth:syscall_entry"
+    assert "missing_attacker_control" in output.read_text()
+
+
+def test_ranking_metrics_track_false_positive_reasons_by_tier() -> None:
+    metrics = compute_ranking_metrics(
+        [
+            {"tier": "A", "outcome": FindingOutcome.VERIFIED, "time_to_verification_seconds": 10},
+            {"tier": "A", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.UNREACHABLE_PATH, "time_to_verification_seconds": 20},
+            {"tier": "B", "outcome": FindingOutcome.REJECTED, "reason": FalsePositiveReason.DUPLICATE},
+            {"tier": "A", "outcome": FindingOutcome.UNVERIFIED, "reason": FalsePositiveReason.UNVERIFIED, "missed_fixture_vulnerabilities": 1},
+        ],
+        tier="A",
+    )
+
+    assert metrics.verified_findings == 1
+    assert metrics.false_positives == 2
+    assert metrics.time_to_verification_seconds == 15.0
+    assert metrics.missed_fixture_vulnerabilities == 1
+    assert metrics.false_positive_reasons["unreachable_path"] == 1
+
+
+def test_scoped_false_positive_demotion_does_not_suppress_class_globally() -> None:
+    learning = record_false_positive_learning(
+        _finding("auth/login.py"),
+        reason=FalsePositiveReason.STATIC_RULE_MISMATCH,
+        evidence="Rule matched a test-only wrapper in this auth path.",
+    )
+
+    same_scope = apply_ranking_calibration(_ranking("auth/login.py"), calibrate_ranking(_ranking("auth/login.py"), [learning]))
+    other_scope = apply_ranking_calibration(_ranking("payments/login.py"), calibrate_ranking(_ranking("payments/login.py"), [learning]))
+
+    assert same_scope.priority < 3.6
+    assert other_scope.priority == 3.6
+
+
+def test_verifier_outcome_feeds_ranking_calibration_and_prompt_constraints() -> None:
+    learning = learning_from_finding_outcome(
+        _finding("auth/login.py"),
+        outcome=FindingOutcome.CONTRADICTED,
+        evidence="Independent verifier showed the risky call is unreachable in production flow.",
+    )
+    assert learning is not None
+
+    calibrated, calibrations = calibrate_rankings([_ranking("auth/login.py")], [learning])
+
+    assert calibrated[0].priority < 3.6
+    assert calibrations[0].retrieval_filter_adjustments == {"syscall_entry": "auth:syscall_entry"}
+    assert "without new evidence" in calibrations[0].prompt_constraints[0]
+
+
+def test_verified_outcome_does_not_create_false_positive_learning() -> None:
+    learning = learning_from_finding_outcome(
+        _finding(),
+        outcome=FindingOutcome.VERIFIED,
+        evidence="Verifier confirmed attacker control and impact.",
+    )
+
+    assert learning is None

tests/test_config.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+from openultrasast.config import load_config, write_resolved_config
+
+
+def test_load_config_reads_toml(tmp_path: Path) -> None:
+    config_path = tmp_path / "openultrasast.toml"
+    config_path.write_text(
+        "\n".join(
+            [
+                "[models]",
+                'ranker = "openrouter/test-ranker"',
+                "[sandbox]",
+                "memory_mb = 1024",
+                "[dynamic]",
+                "enabled = true",
+                'network_scope = ["127.0.0.1:8080"]',
+            ]
+        )
+    )
+
+    config = load_config(config_path)
+
+    assert config.models.ranker == "openrouter/test-ranker"
+    assert config.embeddings.store == "json-local"
+    assert config.sandbox.memory_mb == 1024
+    assert config.dynamic.enabled is True
+    assert config.dynamic.network_scope == ("127.0.0.1:8080",)
+
+
+def test_write_resolved_config_creates_json(tmp_path: Path) -> None:
+    output = tmp_path / "run" / "resolved_config.json"
+
+    write_resolved_config(load_config(None), output)
+
+    assert output.exists()
+    assert '"minimum_report_verified": "static_corroboration"' in output.read_text()

tests/test_findings.py

@@ -0,0 +1,53 @@
+import json
+from pathlib import Path
+
+from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings
+from openultrasast.preprocess import preprocess_repository
+from openultrasast.rank import rank_targets
+from openultrasast.reports import write_markdown_report
+
+
+def test_quick_scan_findings_emit_static_corroboration(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "parser.c").write_text("void parse(char *src) { char dst[8]; strcpy(dst, src); }\n")
+    _, targets = preprocess_repository(repo)
+    rankings = rank_targets(targets)
+
+    findings = quick_scan_findings(repo, targets, rankings)
+
+    assert len(findings) == 1
+    assert findings[0].evidence_level == "static_corroboration"
+    assert findings[0].path == "parser.c"
+    assert findings[0].line == 1
+
+
+def test_findings_and_report_artifacts(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "runner.py").write_text("import subprocess\nsubprocess.run(cmd, shell=True)\n")
+    _, targets = preprocess_repository(repo)
+    findings = quick_scan_findings(repo, targets, rank_targets(targets))
+    findings_path = tmp_path / "findings.json"
+    report_path = tmp_path / "report.md"
+
+    write_findings(findings, findings_path)
+    write_markdown_report(findings, report_path)
+
+    assert json.loads(findings_path.read_text())["findings"][0]["severity"] == "medium"
+    assert "Subprocess shell execution" in report_path.read_text()
+
+
+def test_quick_hunter_prompt_is_bounded() -> None:
+    repo = Path("/repo/example.py")
+    target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None
+    assert target is None
+    class Target:
+        path = "example.py"
+        language = "python"
+        tags = ["auth_boundary"]
+
+    prompt = build_quick_hunter_prompt(Target(), "x" * 5000)  # type: ignore[arg-type]
+
+    assert "example.py" in prompt
+    assert len(prompt) < 4300

tests/test_index.py

@@ -0,0 +1,117 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.index import CodeChunk
+from openultrasast.index import (
+    build_code_chunks,
+    build_retrieval_package,
+    build_vector_index,
+    chunk_text_namespace,
+    index_reuse_key,
+    query_vector_index,
+    read_vector_index,
+    write_vector_index,
+)
+from openultrasast.preprocess import RepoSnapshot, preprocess_repository
+from openultrasast.vectorstore import bakeoff_json_local, bakeoff_payload
+
+
+class FakeEmbeddingClient:
+    def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
+        return [[float(len(text)), 1.0] for text in inputs]
+
+
+def test_build_code_chunks_preserves_metadata_filters(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def login(user):\n    return user\n")
+    _, targets = preprocess_repository(repo)
+
+    chunks = build_code_chunks(repo, targets, max_lines=1)
+
+    assert len(chunks) == 2
+    assert chunks[0].metadata["path"] == "app.py"
+    assert chunks[0].metadata["language"] == "python"
+    assert chunks[0].start_line == 1
+
+
+def test_json_vector_index_round_trips_and_filters(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def login(user):\n    return user\n")
+    snapshot, targets = preprocess_repository(repo)
+    chunks = build_code_chunks(repo, targets)
+    index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="openrouter/embed", client=FakeEmbeddingClient())
+    output = tmp_path / "index.json"
+
+    write_vector_index(index, output)
+    loaded = read_vector_index(output)
+    matches = query_vector_index(loaded, [20.0, 1.0], metadata_filter={"language": "python"})
+
+    assert loaded.embedding_model == "openrouter/embed"
+    assert matches[0][1].path == "app.py"
+    assert json.loads(output.read_text())["store"] == "json-local"
+
+
+def test_vector_index_rejects_embedding_count_mismatch() -> None:
+    class EmptyEmbeddingClient:
+        def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
+            return []
+
+    chunk = CodeChunk(
+        chunk_id="chunk-1",
+        namespace="repo_code",
+        path="app.py",
+        language="python",
+        start_line=1,
+        end_line=1,
+        text="print('x')",
+        metadata={"path": "app.py", "language": "python"},
+    )
+    snapshot = RepoSnapshot(root="/repo", commit=None, file_count=0, languages={})
+    with pytest.raises(ValueError):
+        build_vector_index(snapshot=snapshot, chunks=[chunk], embedding_model="model", client=EmptyEmbeddingClient())
+
+
+def test_namespace_chunks_and_reuse_key_are_metadata_scoped() -> None:
+    chunks = chunk_text_namespace(
+        namespace="skills",
+        path="semgrep.md",
+        text="Use Semgrep for variant mapping.\nKeep provenance.",
+        metadata={"language": "markdown", "vulnerability_class": "mapping"},
+        max_lines=1,
+    )
+    snapshot = RepoSnapshot(root="/repo", commit="abc", file_count=1, languages={"python": 1})
+
+    assert len(chunks) == 2
+    assert chunks[0].namespace == "skills"
+    assert chunks[0].metadata["vulnerability_class"] == "mapping"
+    assert index_reuse_key(snapshot, "embed-a") != index_reuse_key(snapshot, "embed-b")
+
+
+def test_retrieval_package_is_bounded_by_character_budget() -> None:
+    chunks = chunk_text_namespace(namespace="docs", path="README.md", text="alpha\n" + "beta" * 100, max_lines=1)
+    snapshot = RepoSnapshot(root="/repo", commit=None, file_count=1, languages={"markdown": 1})
+    index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient())
+
+    package = build_retrieval_package(role="verifier", index=index, query_embedding=[4.0, 1.0], max_chars=80)
+
+    assert package.role == "verifier"
+    assert package.hits
+    assert package.truncated is True
+
+
+def test_json_local_bakeoff_selects_default_store(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def login(user):\n    return user\n")
+    snapshot, targets = preprocess_repository(repo)
+    chunks = build_code_chunks(repo, targets)
+    index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient())
+
+    result = bakeoff_json_local(index, tmp_path / "vector-index.json", query_embedding=[20.0, 1.0])
+
+    assert result.passed is True
+    assert bakeoff_payload(result)["selected_default"] is True

tests/test_preprocess.py

@@ -0,0 +1,43 @@
+import json
+from pathlib import Path
+
+from openultrasast.preprocess import detect_language, preprocess_repository
+
+
+def test_preprocess_emits_file_targets_with_tags(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / ".gitignore").write_text("ignored.c\n")
+    (repo / "parser.c").write_text(
+        "\n".join(
+            [
+                "int LLVMFuzzerTestOneInput(const unsigned char *data, int size) {",
+                "  parse_frame(data, size);",
+                "  return 0;",
+                "}",
+            ]
+        )
+    )
+    (repo / "auth.py").write_text("def login(session):\n    return open(session)\n")
+    (repo / "ignored.c").write_text("int ignored(void) { return 0; }\n")
+
+    output = tmp_path / "file_targets.json"
+    snapshot, targets = preprocess_repository(repo, output)
+
+    assert snapshot.file_count == 2
+    by_path = {target.path: target for target in targets}
+    assert by_path["parser.c"].language == "c"
+    assert by_path["parser.c"].has_fuzz_entry_point is True
+    assert set(by_path["parser.c"].tags) >= {"memory_unsafe", "parser", "fuzzable"}
+    assert set(by_path["auth.py"].tags) >= {"auth_boundary", "filesystem_entry"}
+
+    payload = json.loads(output.read_text())
+    assert payload["snapshot"]["file_count"] == 2
+    assert [item["path"] for item in payload["file_targets"]] == ["auth.py", "parser.c"]
+
+
+def test_detect_language_uses_shebang(tmp_path: Path) -> None:
+    script = tmp_path / "tool"
+    script.write_text("#!/usr/bin/env python3\nprint('ok')\n")
+
+    assert detect_language(script) == "python"

tests/test_rank.py

@@ -0,0 +1,112 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.preprocess import FileTarget
+from openultrasast.provider.openrouter import OpenRouterError, parse_json_content
+from openultrasast.rank import (
+    RankingError,
+    composite_priority,
+    parse_ranker_response,
+    rank_targets,
+    rank_targets_with_model,
+    write_rankings,
+)
+
+
+class FakeRankerClient:
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
+        self.calls += 1
+        assert model == "openrouter/test-ranker"
+        return {
+            "rankings": [
+                {
+                    "path": "a.py" if self.calls == 1 else "b.py",
+                    "surface": 2,
+                    "influence": 3,
+                    "reachability": 4,
+                    "rationale": "model rationale",
+                    "static_boosts": ["model"],
+                }
+            ]
+        }
+
+
+def test_composite_priority_uses_spec_formula() -> None:
+    assert composite_priority(surface=5, influence=4, reachability=3) == 4.2
+
+
+def test_heuristic_ranking_prioritizes_fuzzable_parser() -> None:
+    targets = [
+        FileTarget(
+            path="parser.c",
+            absolute_path="/repo/parser.c",
+            language="c",
+            loc=10,
+            tags=["memory_unsafe", "parser", "fuzzable"],
+            has_fuzz_entry_point=True,
+        ),
+        FileTarget(
+            path="util.py",
+            absolute_path="/repo/util.py",
+            language="python",
+            loc=5,
+            tags=[],
+            has_fuzz_entry_point=False,
+        ),
+    ]
+
+    rankings = rank_targets(targets)
+
+    assert rankings[0].path == "parser.c"
+    assert rankings[0].surface == 5
+    assert rankings[0].reachability == 4
+
+
+def test_parse_ranker_response_rejects_malformed_scores() -> None:
+    with pytest.raises(RankingError):
+        parse_ranker_response({"rankings": [{"path": "a.py", "surface": 9, "influence": 1, "reachability": 1}]}, "model")
+
+
+def test_rank_targets_with_model_chunks_requests() -> None:
+    targets = [
+        FileTarget(path="a.py", absolute_path="/repo/a.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
+        FileTarget(path="b.py", absolute_path="/repo/b.py", language="python", loc=1, tags=[], has_fuzz_entry_point=False),
+    ]
+    client = FakeRankerClient()
+
+    rankings = rank_targets_with_model(targets, client=client, model="openrouter/test-ranker", chunk_size=1)
+
+    assert client.calls == 2
+    assert {ranking.model_id for ranking in rankings} == {"openrouter/test-ranker"}
+
+
+def test_parse_json_content_accepts_fenced_json() -> None:
+    assert parse_json_content('```json\n{"rankings": []}\n```') == {"rankings": []}
+
+
+def test_parse_json_content_rejects_non_json() -> None:
+    with pytest.raises(OpenRouterError):
+        parse_json_content("not json")
+
+
+def test_write_rankings_emits_artifact(tmp_path: Path) -> None:
+    target = FileTarget(
+        path="auth.py",
+        absolute_path="/repo/auth.py",
+        language="python",
+        loc=5,
+        tags=["auth_boundary"],
+        has_fuzz_entry_point=False,
+    )
+    output = tmp_path / "rank" / "ranking.json"
+
+    write_rankings(rank_targets([target]), output)
+
+    payload = json.loads(output.read_text())
+    assert payload["rankings"][0]["path"] == "auth.py"
+    assert payload["rankings"][0]["priority"] >= 1

tests/test_run.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+from openultrasast.config import load_config
+from openultrasast.run import create_scan_run
+
+
+def test_create_scan_run_writes_directories_and_config(tmp_path: Path) -> None:
+    run = create_scan_run(tmp_path, load_config(None))
+
+    assert run.root.exists()
+    assert (run.root / "preprocess").is_dir()
+    assert (run.root / "resolved_config.json").is_file()
+    assert run.root.parent.name == "runs"