test(learning-harness): apply the group 1 review suggestions

A mechanism claimed by two families now fails as loudly as a weakness claimed
twice, with a named offender. Two behaviours that were correct but unasserted
are pinned: the obligations tool refuses a path outside the repository, and a
provider that refuses to build degrades to no endpoint instead of raising into
a scan. The TypeScript parse assertion is now exact rather than a quarter-of-
the-slice allowance, which surfaces one real corpus defect: a TSX excerpt
harvested as a truncated destructuring parameter list is not valid TypeScript
on its own, and belongs to the re-harvest rather than to the grammar.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 09:40:24 +02:00
co-authored by Claude Opus 5
parent d05c516a10
commit 17fc18c785
5 changed files with 29 additions and 1 deletions
+7
View File
@@ -99,6 +99,7 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
raise FamiliesError(f"{source}: version is required")
families: list[Family] = []
claimed: dict[str, str] = {}
claimed_mechanisms: dict[str, str] = {}
for entry in payload.get("family", []):
if not isinstance(entry, dict):
raise FamiliesError(f"{source}: every [[family]] entry must be a table")
@@ -109,6 +110,12 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
if cwe in claimed:
raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}")
claimed[cwe] = family.id
for mechanism in sorted(family.mechanisms):
if mechanism in claimed_mechanisms:
raise FamiliesError(
f"{source}: mechanism {mechanism} is claimed by both {claimed_mechanisms[mechanism]!r} and {family.id!r}"
)
claimed_mechanisms[mechanism] = family.id
families.append(family)
_check_parents(source, families)
if require_all and tuple(family.id for family in families) != FAMILY_IDS:
+2
View File
@@ -65,6 +65,8 @@ def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path)
assert "identity_constraint" in {str(row["missing"]) for row in rows}
assert all("text" not in row and "content" not in row for row in rows)
assert obligations(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
obligations(repo, path="../outside.py", function="leaky")
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
+12
View File
@@ -67,6 +67,18 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke
assert resolve_chat_endpoint(ResolvedConfig()) is None
def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
from openultrasast.learning.endpoint import resolve_chat_endpoint
_endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key")
def _refuse() -> provider.OpenRouterChatClient:
raise provider.OpenRouterError("no key after all")
monkeypatch.setattr(provider.OpenRouterChatClient, "from_env", staticmethod(_refuse))
assert resolve_chat_endpoint(ResolvedConfig()) is None # a caller degrades; nothing raises into a scan
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
from openultrasast.config import load_config
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
+5
View File
@@ -88,6 +88,10 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
(GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"),
(GOOD.replace('description = "x"\n', ""), "description"),
(GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"),
(
GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = []\nmechanisms = ["source_reaches_sink"]\nverifier = "canary"\n',
"source_reaches_sink",
),
(GOOD.replace('version = "1"\n', ""), "version"),
(GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"),
],
@@ -99,6 +103,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
"duplicate_cwe",
"no_description",
"cwe_claimed_twice",
"mechanism_claimed_twice",
"no_version",
"unknown_parent",
],
+3 -1
View File
@@ -46,4 +46,6 @@ def test_every_vendored_typescript_pair_parses_on_both_sides() -> None:
elif not ir.parse_ok:
failed.append(f"{case.name}:{side}")
assert unsupported == []
assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar
# One excerpt is a truncated destructuring parameter list, so it is not valid TypeScript on its own;
# that is a corpus defect for the re-harvest, not a grammar gap. Pinned exactly so a regression shows.
assert failed == ["lensops-kubernetesmanagement-fe4910:vuln", "lensops-kubernetesmanagement-fe4910:fixed"], failed