mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
test(learning-harness): apply the group 1 review suggestions
A mechanism claimed by two families now fails as loudly as a weakness claimed twice, with a named offender. Two behaviours that were correct but unasserted are pinned: the obligations tool refuses a path outside the repository, and a provider that refuses to build degrades to no endpoint instead of raising into a scan. The TypeScript parse assertion is now exact rather than a quarter-of- the-slice allowance, which surfaces one real corpus defect: a TSX excerpt harvested as a truncated destructuring parameter list is not valid TypeScript on its own, and belongs to the re-harvest rather than to the grammar. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
d05c516a10
commit
17fc18c785
@@ -99,6 +99,7 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
|
||||
raise FamiliesError(f"{source}: version is required")
|
||||
families: list[Family] = []
|
||||
claimed: dict[str, str] = {}
|
||||
claimed_mechanisms: dict[str, str] = {}
|
||||
for entry in payload.get("family", []):
|
||||
if not isinstance(entry, dict):
|
||||
raise FamiliesError(f"{source}: every [[family]] entry must be a table")
|
||||
@@ -109,6 +110,12 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
|
||||
if cwe in claimed:
|
||||
raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}")
|
||||
claimed[cwe] = family.id
|
||||
for mechanism in sorted(family.mechanisms):
|
||||
if mechanism in claimed_mechanisms:
|
||||
raise FamiliesError(
|
||||
f"{source}: mechanism {mechanism} is claimed by both {claimed_mechanisms[mechanism]!r} and {family.id!r}"
|
||||
)
|
||||
claimed_mechanisms[mechanism] = family.id
|
||||
families.append(family)
|
||||
_check_parents(source, families)
|
||||
if require_all and tuple(family.id for family in families) != FAMILY_IDS:
|
||||
|
||||
@@ -65,6 +65,8 @@ def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path)
|
||||
assert "identity_constraint" in {str(row["missing"]) for row in rows}
|
||||
assert all("text" not in row and "content" not in row for row in rows)
|
||||
assert obligations(repo, path="app.py", function="helper") == []
|
||||
with pytest.raises(PathEscapesRepo):
|
||||
obligations(repo, path="../outside.py", function="leaky")
|
||||
|
||||
|
||||
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
|
||||
|
||||
@@ -67,6 +67,18 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke
|
||||
assert resolve_chat_endpoint(ResolvedConfig()) is None
|
||||
|
||||
|
||||
def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint
|
||||
|
||||
_endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key")
|
||||
|
||||
def _refuse() -> provider.OpenRouterChatClient:
|
||||
raise provider.OpenRouterError("no key after all")
|
||||
|
||||
monkeypatch.setattr(provider.OpenRouterChatClient, "from_env", staticmethod(_refuse))
|
||||
assert resolve_chat_endpoint(ResolvedConfig()) is None # a caller degrades; nothing raises into a scan
|
||||
|
||||
|
||||
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
from openultrasast.config import load_config
|
||||
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
|
||||
|
||||
@@ -88,6 +88,10 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
|
||||
(GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"),
|
||||
(GOOD.replace('description = "x"\n', ""), "description"),
|
||||
(GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"),
|
||||
(
|
||||
GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = []\nmechanisms = ["source_reaches_sink"]\nverifier = "canary"\n',
|
||||
"source_reaches_sink",
|
||||
),
|
||||
(GOOD.replace('version = "1"\n', ""), "version"),
|
||||
(GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"),
|
||||
],
|
||||
@@ -99,6 +103,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
|
||||
"duplicate_cwe",
|
||||
"no_description",
|
||||
"cwe_claimed_twice",
|
||||
"mechanism_claimed_twice",
|
||||
"no_version",
|
||||
"unknown_parent",
|
||||
],
|
||||
|
||||
@@ -46,4 +46,6 @@ def test_every_vendored_typescript_pair_parses_on_both_sides() -> None:
|
||||
elif not ir.parse_ok:
|
||||
failed.append(f"{case.name}:{side}")
|
||||
assert unsupported == []
|
||||
assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar
|
||||
# One excerpt is a truncated destructuring parameter list, so it is not valid TypeScript on its own;
|
||||
# that is a corpus defect for the re-harvest, not a grammar gap. Pinned exactly so a regression shows.
|
||||
assert failed == ["lensops-kubernetesmanagement-fe4910:vuln", "lensops-kubernetesmanagement-fe4910:fixed"], failed
|
||||
|
||||
Reference in New Issue
Block a user