mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
pre-push(17.2): name every skipped check in plain words
One terminal line per distinct coverage reason (push/explain.py): hex-encoded paths decoded, per-question sha256 tokens dropped and counted, unknown reasons still printed; the JSON artifact keeps the raw values. A changed language with no engine frontend and no quick rules is recorded and printed as "language not covered: <lang> (...)", in pre-push and in `ousast scan`. The scan report renders the eight degradations cli.py recorded but never showed, and any reason without a sentence. The hook forwards OUSAST_COMPARISON_BASE. Requirement 9.2. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bf07cd4d1d
commit
f555dee640
@@ -0,0 +1,88 @@
|
||||
"""Plain-language skip lines (pre-push 17.2, Requirement 9.2): nothing hidden, nothing opaque."""
|
||||
|
||||
import re
|
||||
from dataclasses import replace
|
||||
|
||||
from test_push_report import budget, report_with
|
||||
|
||||
from openultrasast.push.explain import explain, language_reason
|
||||
from openultrasast.push.report import deliver_report
|
||||
|
||||
# Coverage reasons recorded by the 2026-10-02 audit's live runs, verbatim in shape.
|
||||
AUDIT = (
|
||||
"7265636f7264732e7079:ambiguous_line_correspondence",
|
||||
"base_counterpart_unresolved",
|
||||
"cpg_build_failed",
|
||||
"frontend_unsupported",
|
||||
"analysis_incomplete",
|
||||
"base_comparison_unavailable",
|
||||
"eligibility:eligibility_provenance_mismatch",
|
||||
*(f"dynamic_external_or_depth_context_unresolved:contributor-scan:{i:064x}" for i in range(20)),
|
||||
"resolution:missing_base",
|
||||
"admission_failed:TimeoutError",
|
||||
"base_incomplete",
|
||||
"comparison_unknown",
|
||||
)
|
||||
|
||||
|
||||
def test_audit_reasons_become_one_plain_line_each():
|
||||
lines = explain(AUDIT, deadline=30)
|
||||
text = "\n".join(lines)
|
||||
assert "records.py" in text and "7265636f7264732e7079" not in text
|
||||
assert not re.search(r"[0-9a-f]{64}", text)
|
||||
assert "20 engine check(s) depend on dynamic or external code" in text
|
||||
assert "Joern" in text and "new branch: the remote has no base" in text
|
||||
assert "the admission step failed (TimeoutError)" in text
|
||||
assert "analysis_incomplete" not in text and "analysis incomplete" not in text
|
||||
# One line per distinct kind: twenty hashed reasons are one line, comparison_unknown folds into its cause.
|
||||
assert len(lines) == 10
|
||||
assert sum("could not be established" in line for line in lines) == 1
|
||||
|
||||
|
||||
def test_unknown_reason_is_still_printed():
|
||||
assert explain(("some_future_reason",)) == ["some future reason (see the details file)"]
|
||||
|
||||
|
||||
def test_deadline_line_names_the_deadline_and_the_background_option():
|
||||
(line,) = explain(("deadline_exhausted",), deadline=30)
|
||||
assert "30 s" in line and "background" in line
|
||||
|
||||
|
||||
def test_language_not_covered_is_named_with_counts():
|
||||
(line,) = explain((language_reason("go", 1, 12),))
|
||||
assert line.startswith("language not covered: go (1 changed file(s), 12 in the repository)")
|
||||
|
||||
|
||||
def test_terminal_shows_skip_lines_while_artifact_keeps_raw_reasons(tmp_path):
|
||||
import json
|
||||
|
||||
report = report_with(0)
|
||||
raw = ("7265636f7264732e7079:ambiguous_line_correspondence", f"dynamic_external_or_depth_context_unresolved:contributor-scan:{1:064x}")
|
||||
report = replace(
|
||||
report,
|
||||
admission=replace(report.admission, coverage_reasons=raw),
|
||||
result=replace(report.result, coverage_status="incomplete"),
|
||||
)
|
||||
delivered = deliver_report(report, tmp_path / "run.json", execution_budget=budget())
|
||||
assert "Skipped: changed lines could not be matched one-to-one to base lines in records.py" in delivered.text
|
||||
assert "7265636f" not in delivered.text and f"{1:064x}" not in delivered.text
|
||||
assert delivered.text.count("Notice:") == 1
|
||||
assert tuple(json.loads(delivered.artifact.read_text())["admission"]["coverage_reasons"]) == raw
|
||||
|
||||
|
||||
def test_replay_names_a_changed_language_nothing_covers(tmp_path):
|
||||
import json
|
||||
|
||||
from test_push_runner import history
|
||||
|
||||
from openultrasast.cpg.backend import NullBackend
|
||||
from openultrasast.push.runner import replay
|
||||
|
||||
root, base, _, git = history(tmp_path)
|
||||
(root / "run.go").write_text("package main\n")
|
||||
git("add", "run.go")
|
||||
git("commit", "-m", "go")
|
||||
head = git("rev-parse", "HEAD")
|
||||
delivery = replay(root, base=base, head=head, artifact=tmp_path / "r.json", backend=NullBackend())
|
||||
assert "Skipped: language not covered: go (1 changed file(s), 1 in the repository)" in delivery.text
|
||||
assert "language_not_covered:go:1:1" in json.loads((tmp_path / "r.json").read_text())["admission"]["coverage_reasons"]
|
||||
@@ -654,3 +654,46 @@ def test_a_clean_scan_states_no_coverage_gaps(tmp_path) -> None: # type: ignore
|
||||
degradations=[],
|
||||
)
|
||||
assert "### What could not be analysed" not in path.read_text()
|
||||
|
||||
|
||||
def test_every_recorded_scan_degradation_is_rendered(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||||
"""pre-push 17.2 / Req 9.2: the eight degradations cli.py records and the report used to drop."""
|
||||
from openultrasast.reports import write_markdown_report
|
||||
|
||||
reasons = [
|
||||
{"stage": "model", "reason": "learning_endpoint_unavailable"},
|
||||
{"stage": "model", "reason": "cpg_unavailable"},
|
||||
{"stage": "overlay", "reason": "facts_unavailable", "detail": "missing table"},
|
||||
{"stage": "obligations", "reason": "facts_unavailable"},
|
||||
{"stage": "obligations", "reason": "policy_invalid", "detail": "bad toml"},
|
||||
{"stage": "model", "reason": "model_withheld_execution_budget"},
|
||||
{"stage": "budget", "reason": "max_findings_exceeded", "requested": 5, "actual": 9},
|
||||
{"stage": "map", "reason": "hunter_model_unavailable"},
|
||||
{"stage": "regress", "reason": "sandbox_unavailable"},
|
||||
{"stage": "x", "reason": "not_yet_described"},
|
||||
]
|
||||
path = tmp_path / "report.md"
|
||||
write_markdown_report([], path, [], degradations=reasons)
|
||||
text = path.read_text()
|
||||
for needle in (
|
||||
"No model endpoint is configured",
|
||||
"Joern engine is not installed",
|
||||
"fact tables for the `overlay` stage",
|
||||
"fact tables for the `obligations` stage",
|
||||
"`obligations` policy could not be loaded",
|
||||
"time budget leaves no room",
|
||||
"first 5 of 9 findings",
|
||||
"needs a model and none is available",
|
||||
"Docker is not available",
|
||||
"`not_yet_described`",
|
||||
):
|
||||
assert needle in text, needle
|
||||
|
||||
|
||||
def test_quick_scan_names_a_language_nothing_covers(tmp_path, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
"""pre-push 17.2 / Req 9.2: a Go-only tree is not a silent zero."""
|
||||
(tmp_path / "run.go").write_text('package main\n\nimport "os/exec"\n\nfunc run(c string) { exec.Command("sh", "-c", c).Run() }\n')
|
||||
assert main(["scan", str(tmp_path), "--mode", "quick"]) == 0
|
||||
assert "notice: language not covered: go (1 files)" in capsys.readouterr().out
|
||||
report = next((tmp_path / ".openultrasast" / "runs").iterdir()) / "report.md"
|
||||
assert "Not a clean result for go (1 files)" in report.read_text()
|
||||
|
||||
Reference in New Issue
Block a user