feat(harness): add runtime and reachability mapping

2026-10-17 12:09:27 +00:00 · 2026-06-18 07:57:06 +02:00
parent 33757db14a
commit f8c690999a
16 changed files with 1339 additions and 53 deletions
@@ -25,6 +25,10 @@ def _finding(path: str = "auth/login.py") -> StaticFinding:
        evidence_level="static_corroboration",
        rationale="pattern matched",
        line=1,
+        function_name=None,
+        reachability_status="unknown",
+        reachability_evidence=[],
+        reachability_conditions=[],
        tags=["syscall_entry"],
        ranking_priority=4.0,
    )
@@ -2,6 +2,7 @@ import json
 from pathlib import Path

 from openultrasast.findings import build_quick_hunter_prompt, quick_scan_findings, write_findings
+from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
 from openultrasast.preprocess import preprocess_repository
 from openultrasast.rank import rank_targets
 from openultrasast.reports import write_markdown_report
@@ -38,6 +39,37 @@ def test_findings_and_report_artifacts(tmp_path: Path) -> None:
    assert "Subprocess shell execution" in report_path.read_text()


+def test_findings_inherit_only_matching_function_level_reachability(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "@app.route('/admin')\n"
+        "@login_required\n"
+        "def admin_upload():\n"
+        "    if feature_flags.enabled('dangerous_upload'):\n"
+        "        return eval(request.data)\n"
+        "    return 'disabled'\n"
+        "\n"
+        "def helper(user_input):\n"
+        "    return exec(user_input)\n"
+    )
+    _, targets = preprocess_repository(repo)
+    targets = attach_reachability_hints(targets, analyze_entry_points(repo, targets))
+
+    findings = quick_scan_findings(repo, targets, rank_targets(targets))
+
+    reachable = next(finding for finding in findings if finding.line == 5)
+    unreachable = next(finding for finding in findings if finding.line == 9)
+
+    assert reachable.function_name == "admin_upload"
+    assert reachable.reachability_status == "reachable"
+    assert reachable.reachability_evidence[0]["access_level"] == "authenticated"
+    assert reachable.reachability_conditions == ["feature_flags.enabled('dangerous_upload')"]
+    assert unreachable.function_name is None
+    assert unreachable.reachability_status == "unknown"
+    assert unreachable.ranking_priority <= 1.5
+
+
 def test_quick_hunter_prompt_is_bounded() -> None:
    repo = Path("/repo/example.py")
    target = next(iter(preprocess_repository(repo.parent)[1]), None) if repo.parent.exists() else None
@@ -0,0 +1,104 @@
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from openultrasast.config import load_config
+from openultrasast.harness import (
+    HarnessContractError,
+    HarnessEvent,
+    HarnessRuntime,
+    HarnessTraceWriter,
+    ProcessorSpec,
+    write_harness_config,
+)
+
+
+@dataclass
+class RecordingProcessor:
+    spec: ProcessorSpec
+
+    def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
+        return {"seen_stage": event.stage}
+
+
+@dataclass
+class BadWriteProcessor:
+    spec: ProcessorSpec
+
+    def handle(self, event: HarnessEvent, state: dict[str, Any]) -> dict[str, Any]:
+        return {"undeclared": event.event_type}
+
+
+def test_runtime_emits_stage_events_and_processor_state(tmp_path: Path) -> None:
+    trace_path = tmp_path / "events.jsonl"
+    processor = RecordingProcessor(
+        ProcessorSpec(name="recorder", version="1", handles=("stage_start",), writes=("seen_stage",))
+    )
+    runtime = HarnessRuntime(
+        scan_id="scan-1",
+        config=load_config(None),
+        trace_writer=HarnessTraceWriter(trace_path),
+        processors=[processor],
+    )
+
+    result = runtime.run_stage("preprocess", lambda: "ok")
+
+    events = [json.loads(line) for line in trace_path.read_text().splitlines()]
+    assert result == "ok"
+    assert runtime.state["seen_stage"] == "preprocess"
+    assert [event["event_type"] for event in events] == ["stage_start", "stage_start", "stage_end"]
+    assert events[1]["payload"] == {"changed_state": ["seen_stage"], "processor": "recorder"}
+
+
+def test_strict_contract_mode_rejects_undeclared_writes(tmp_path: Path) -> None:
+    processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
+    runtime = HarnessRuntime(
+        scan_id="scan-1",
+        config=load_config(None),
+        trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
+        processors=[processor],
+    )
+
+    with pytest.raises(HarnessContractError, match="undeclared writes"):
+        runtime.emit("stage_start", "rank", {})
+
+
+def test_warn_contract_mode_records_degradation(tmp_path: Path) -> None:
+    processor = BadWriteProcessor(ProcessorSpec(name="bad", version="1", handles=("stage_start",), writes=()))
+    runtime = HarnessRuntime(
+        scan_id="scan-1",
+        config=load_config(None),
+        trace_writer=HarnessTraceWriter(tmp_path / "events.jsonl"),
+        processors=[processor],
+        contract_mode="warn",
+    )
+
+    runtime.emit("stage_start", "rank", {})
+
+    assert "undeclared writes" in runtime.state["degradations"][0]
+    assert '"status": "degraded"' in (tmp_path / "events.jsonl").read_text()
+
+
+def test_harness_config_serializes_models_processors_and_prompt_hashes(tmp_path: Path) -> None:
+    config_path = tmp_path / "openultrasast.toml"
+    config_path.write_text("[models]\nranker = \"openrouter/ranker\"\n")
+    processor = RecordingProcessor(
+        ProcessorSpec(name="recorder", version="2026-01-01", handles=("stage_start",), writes=("seen_stage",))
+    )
+    output = tmp_path / "harness.json"
+
+    write_harness_config(
+        config=load_config(config_path),
+        processors=[processor],
+        contract_mode="strict",
+        path=output,
+        prompts={"ranker": "score files"},
+    )
+
+    payload = json.loads(output.read_text())
+    assert payload["model_roles"]["ranker"] == "openrouter/ranker"
+    assert payload["processor_versions"] == {"recorder": "2026-01-01"}
+    assert len(payload["prompt_hashes"]["ranker"]) == 64
@@ -0,0 +1,215 @@
+import json
+from pathlib import Path
+
+from openultrasast.mapping import (
+    analyze_entry_points,
+    attach_reachability_hints,
+    attach_static_hints,
+    codeql_mapping_tasks,
+    differential_mapping_record,
+    ingest_sarif,
+    semgrep_mapping_tasks,
+    sharp_edge_record,
+    verifier_evidence_candidates,
+    write_entry_points,
+    write_static_hints,
+)
+from openultrasast.preprocess import preprocess_repository
+
+
+def test_ingest_sarif_normalizes_semgrep_hints(tmp_path: Path) -> None:
+    sarif = tmp_path / "semgrep.sarif"
+    sarif.write_text(json.dumps(_sarif_payload("Semgrep", "python.lang.security.audit.eval", "app.py")))
+
+    hints = ingest_sarif(sarif)
+
+    assert len(hints) == 1
+    assert hints[0].analyzer == "semgrep"
+    assert hints[0].rule_id == "python.lang.security.audit.eval"
+    assert hints[0].path == "app.py"
+    assert hints[0].start_line == 7
+    assert hints[0].severity == "high"
+    assert hints[0].provenance == "sarif:semgrep:python.lang.security.audit.eval"
+
+
+def test_static_hints_attach_to_file_targets_and_evidence_candidates(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("eval(user_input)\n")
+    hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "py/code-injection", "app.py"))
+    _, targets = preprocess_repository(repo, static_hints=hints)
+
+    enriched = attach_static_hints(targets, hints)
+    evidence = verifier_evidence_candidates(hints, path="app.py")
+
+    assert enriched[0].static_hints[0]["analyzer"] == "codeql"
+    assert evidence == [
+        {
+            "source": "codeql",
+            "rule_id": "py/code-injection",
+            "path": "app.py",
+            "line": 7,
+            "severity": "high",
+            "message": "Potential injection",
+            "provenance": "sarif:codeql:py/code-injection",
+        }
+    ]
+
+
+def test_mapping_task_records_cover_disciplines(tmp_path: Path) -> None:
+    semgrep_hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "parser.c"))
+    codeql_hints = ingest_sarif(_write_sarif(tmp_path, "CodeQL", "cpp/overflow", "parser.c"))
+
+    assert semgrep_mapping_tasks(semgrep_hints)[0].task == "pattern_variant_review"
+    assert codeql_mapping_tasks(codeql_hints)[0].task == "source_sink_sanitizer_path_review"
+    assert differential_mapping_record(
+        "parser.c",
+        change_kind="modified_function",
+        trust_boundary="network_input",
+        rationale="Parser changed behind a socket boundary.",
+    ).trust_boundary == "network_input"
+    assert sharp_edge_record(
+        "config.py",
+        category="insecure_default",
+        surface="debug mode",
+        rationale="Debug default changes production risk.",
+    ).category == "insecure_default"
+
+
+def test_write_static_hints_artifact(tmp_path: Path) -> None:
+    hints = ingest_sarif(_write_sarif(tmp_path, "Semgrep", "sg.rule", "app.py"))
+    output = tmp_path / "mapping" / "static_hints.json"
+
+    write_static_hints(hints, output)
+
+    assert json.loads(output.read_text())["static_hints"][0]["analyzer"] == "semgrep"
+
+
+def test_entry_point_mapping_classifies_routes_and_attaches_reachability(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("@app.route('/upload')\ndef upload():\n    return parse(request.data)\n")
+    _, targets = preprocess_repository(repo)
+
+    entry_points = analyze_entry_points(repo, targets)
+    enriched = attach_reachability_hints(targets, entry_points)
+
+    assert entry_points[0].kind == "route"
+    assert entry_points[0].access_level == "public"
+    assert enriched[0].reachability_hints[0]["trust_boundary"] == "http_request"
+
+
+def test_entry_point_mapping_distinguishes_authenticated_routes_and_conditions(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "@app.route('/admin')\n"
+        "@login_required\n"
+        "def admin_upload():\n"
+        "    if feature_flags.enabled('dangerous_upload'):\n"
+        "        return eval(request.data)\n"
+        "    return 'disabled'\n"
+    )
+    _, targets = preprocess_repository(repo)
+
+    entry_points = analyze_entry_points(repo, targets)
+
+    route = next(entry_point for entry_point in entry_points if entry_point.kind == "route")
+    assert route.function_name == "admin_upload"
+    assert route.line == 3
+    assert route.end_line == 6
+    assert route.access_level == "authenticated"
+    assert route.access_evidence == ["login_required"]
+    assert route.conditions == ["feature_flags.enabled('dangerous_upload')"]
+
+
+def test_entry_point_mapping_covers_solidity_state_changing_access(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "Vault.sol").write_text(
+        "contract Vault {\n"
+        "  function withdraw(uint256 amount) external onlyOwner { }\n"
+        "  function balance() external view returns (uint256) { return 0; }\n"
+        "}\n"
+    )
+    _, targets = preprocess_repository(repo)
+
+    entry_points = analyze_entry_points(repo, targets)
+
+    assert len(entry_points) == 1
+    assert entry_points[0].name == "withdraw"
+    assert entry_points[0].access_level == "role-restricted"
+
+
+def test_entry_point_mapping_records_solidity_feature_conditions(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "Vault.sol").write_text(
+        "contract Vault {\n"
+        "  function withdraw(uint256 amount) external onlyOwner whenNotPaused { }\n"
+        "}\n"
+    )
+    _, targets = preprocess_repository(repo)
+
+    entry_points = analyze_entry_points(repo, targets)
+
+    assert entry_points[0].access_level == "role-restricted"
+    assert entry_points[0].access_evidence == ["onlyOwner"]
+    assert entry_points[0].conditions == ["whenNotPaused"]
+
+
+def test_write_entry_points_artifact(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "cli.py").write_text("if __name__ == '__main__':\n    main()\n")
+    _, targets = preprocess_repository(repo)
+    output = tmp_path / "mapping" / "entry_points.json"
+
+    write_entry_points(analyze_entry_points(repo, targets), output)
+
+    payload = json.loads(output.read_text())
+    assert payload["entry_points"][0]["kind"] == "cli"
+
+
+def _write_sarif(tmp_path: Path, analyzer: str, rule_id: str, result_path: str) -> Path:
+    path = tmp_path / f"{analyzer.lower()}.sarif"
+    path.write_text(json.dumps(_sarif_payload(analyzer, rule_id, result_path)))
+    return path
+
+
+def _sarif_payload(analyzer: str, rule_id: str, result_path: str) -> dict[str, object]:
+    return {
+        "version": "2.1.0",
+        "runs": [
+            {
+                "tool": {
+                    "driver": {
+                        "name": analyzer,
+                        "rules": [
+                            {
+                                "id": rule_id,
+                                "name": "Injection rule",
+                                "properties": {"security-severity": "8.1"},
+                            }
+                        ],
+                    }
+                },
+                "results": [
+                    {
+                        "ruleId": rule_id,
+                        "level": "error",
+                        "message": {"text": "Potential injection"},
+                        "partialFingerprints": {"primaryLocationLineHash": "abc123"},
+                        "locations": [
+                            {
+                                "physicalLocation": {
+                                    "artifactLocation": {"uri": result_path},
+                                    "region": {"startLine": 7},
+                                }
+                            }
+                        ],
+                    }
+                ],
+            }
+        ],
+    }
@@ -110,3 +110,26 @@ def test_write_rankings_emits_artifact(tmp_path: Path) -> None:
    payload = json.loads(output.read_text())
    assert payload["rankings"][0]["path"] == "auth.py"
    assert payload["rankings"][0]["priority"] >= 1
+
+
+def test_reachability_hints_raise_ranking_priority() -> None:
+    target = FileTarget(
+        path="routes.py",
+        absolute_path="/repo/routes.py",
+        language="python",
+        loc=5,
+        tags=[],
+        has_fuzz_entry_point=False,
+        reachability_hints=[
+            {
+                "kind": "route",
+                "access_level": "public",
+                "trust_boundary": "http_request",
+            }
+        ],
+    )
+
+    ranking = rank_targets([target])[0]
+
+    assert ranking.reachability == 5
+    assert "entry_point_reachable" in ranking.static_boosts