mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Fable 5.1
16d9ef7110
openultrasast-scan: evidence ladder as verification.py defines it, with the levels nothing produces today; --mode prerequisites, --fail-on, recorded degradations, run artifacts, ousast mcp; retired [evidence] threshold language removed. openultrasast-triage: reasons verified against calibration.py; deterministic fusion; adjudications as labels, plane verdicts as features, rule changes through improve or a registered experiment; fix-audit merged as a fix-and-re-scan section. kiro: spec-quick and spec-batch removed; validate-gap and validate-design merged into spec-design (--gap, --review; the review gate stays); steering-custom merged into steering; core steering names overview.md, roadmap.md, safety-net.md; instrument rule in debug and verify-completion; repo gates named in validate-impl. AGENTS.md, README skills paragraph and tests/test_agent_skills.py (.agents/skills, two product skills, drift tests) updated. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
from pathlib import Path
|
|
|
|
SKILLS_ROOT = Path(".agents/skills") # the plural directory OpenCode searches by default
|
|
|
|
|
|
def test_project_agent_skills_have_required_frontmatter() -> None:
|
|
# Scope to the project's own skills; third-party tooling (e.g. kiro-*) lives
|
|
# alongside them and uses a different frontmatter convention.
|
|
skill_files = sorted(SKILLS_ROOT.glob("openultrasast-*/SKILL.md"))
|
|
|
|
assert {path.parent.name for path in skill_files} == {
|
|
"openultrasast-scan",
|
|
"openultrasast-triage",
|
|
}
|
|
|
|
for skill_file in skill_files:
|
|
text = skill_file.read_text()
|
|
header = text.split("---", 2)[1]
|
|
assert f"name: {skill_file.parent.name}" in header
|
|
assert "description: " in header
|
|
assert "Use when" in header
|
|
|
|
|
|
def test_scan_skill_states_the_evidence_ladder_as_the_code_defines_it() -> None:
|
|
# The skill must not drift from verification.py again: every level, in the code's order, and the
|
|
# retired threshold language gone.
|
|
text = (SKILLS_ROOT / "openultrasast-scan" / "SKILL.md").read_text()
|
|
levels = [
|
|
"suspicion",
|
|
"static_corroboration",
|
|
"crash_reproduced",
|
|
"root_cause_explained",
|
|
"exploit_demonstrated",
|
|
"patch_validated",
|
|
]
|
|
ladder = text[text.index("## Evidence ladder") : text.index("## Run artifacts")]
|
|
positions = [ladder.index(f"`{level}`") for level in levels]
|
|
assert positions == sorted(positions)
|
|
assert "configured report threshold" not in text
|
|
current_cli = (
|
|
"--mode quick|standard|deep",
|
|
"--fail-on",
|
|
"cpg_unavailable",
|
|
"sandbox_unavailable",
|
|
"report.sarif",
|
|
"harness.json",
|
|
"ousast mcp",
|
|
)
|
|
for flag in current_cli:
|
|
assert flag in text, flag
|
|
|
|
|
|
def test_triage_skill_lists_the_false_positive_reasons_exactly() -> None:
|
|
from openultrasast.calibration import FalsePositiveReason
|
|
|
|
text = (SKILLS_ROOT / "openultrasast-triage" / "SKILL.md").read_text()
|
|
listed = {line.strip()[3:-1] for line in text.splitlines() if line.startswith("- `") and line.strip().endswith("`")}
|
|
assert listed == {reason.value for reason in FalsePositiveReason}
|
|
assert "patch_validated" in text and "Nothing in the tool records `patch_validated`" in text
|