test(learning-harness): apply the group 1 review suggestions

A mechanism claimed by two families now fails as loudly as a weakness claimed
twice, with a named offender. Two behaviours that were correct but unasserted
are pinned: the obligations tool refuses a path outside the repository, and a
provider that refuses to build degrades to no endpoint instead of raising into
a scan. The TypeScript parse assertion is now exact rather than a quarter-of-
the-slice allowance, which surfaces one real corpus defect: a TSX excerpt
harvested as a truncated destructuring parameter list is not valid TypeScript
on its own, and belongs to the re-harvest rather than to the grammar.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 09:40:24 +02:00
co-authored by Claude Opus 5
parent d05c516a10
commit 17fc18c785
5 changed files with 29 additions and 1 deletions
+7
View File
@@ -99,6 +99,7 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
raise FamiliesError(f"{source}: version is required") raise FamiliesError(f"{source}: version is required")
families: list[Family] = [] families: list[Family] = []
claimed: dict[str, str] = {} claimed: dict[str, str] = {}
claimed_mechanisms: dict[str, str] = {}
for entry in payload.get("family", []): for entry in payload.get("family", []):
if not isinstance(entry, dict): if not isinstance(entry, dict):
raise FamiliesError(f"{source}: every [[family]] entry must be a table") raise FamiliesError(f"{source}: every [[family]] entry must be a table")
@@ -109,6 +110,12 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
if cwe in claimed: if cwe in claimed:
raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}") raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}")
claimed[cwe] = family.id claimed[cwe] = family.id
for mechanism in sorted(family.mechanisms):
if mechanism in claimed_mechanisms:
raise FamiliesError(
f"{source}: mechanism {mechanism} is claimed by both {claimed_mechanisms[mechanism]!r} and {family.id!r}"
)
claimed_mechanisms[mechanism] = family.id
families.append(family) families.append(family)
_check_parents(source, families) _check_parents(source, families)
if require_all and tuple(family.id for family in families) != FAMILY_IDS: if require_all and tuple(family.id for family in families) != FAMILY_IDS:
+2
View File
@@ -65,6 +65,8 @@ def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path)
assert "identity_constraint" in {str(row["missing"]) for row in rows} assert "identity_constraint" in {str(row["missing"]) for row in rows}
assert all("text" not in row and "content" not in row for row in rows) assert all("text" not in row and "content" not in row for row in rows)
assert obligations(repo, path="app.py", function="helper") == [] assert obligations(repo, path="app.py", function="helper") == []
with pytest.raises(PathEscapesRepo):
obligations(repo, path="../outside.py", function="leaky")
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None: def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
+12
View File
@@ -67,6 +67,18 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke
assert resolve_chat_endpoint(ResolvedConfig()) is None assert resolve_chat_endpoint(ResolvedConfig()) is None
def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
from openultrasast.learning.endpoint import resolve_chat_endpoint
_endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key")
def _refuse() -> provider.OpenRouterChatClient:
raise provider.OpenRouterError("no key after all")
monkeypatch.setattr(provider.OpenRouterChatClient, "from_env", staticmethod(_refuse))
assert resolve_chat_endpoint(ResolvedConfig()) is None # a caller degrades; nothing raises into a scan
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
from openultrasast.config import load_config from openultrasast.config import load_config
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
+5
View File
@@ -88,6 +88,10 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
(GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"), (GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"),
(GOOD.replace('description = "x"\n', ""), "description"), (GOOD.replace('description = "x"\n', ""), "description"),
(GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"), (GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"),
(
GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = []\nmechanisms = ["source_reaches_sink"]\nverifier = "canary"\n',
"source_reaches_sink",
),
(GOOD.replace('version = "1"\n', ""), "version"), (GOOD.replace('version = "1"\n', ""), "version"),
(GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"), (GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"),
], ],
@@ -99,6 +103,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
"duplicate_cwe", "duplicate_cwe",
"no_description", "no_description",
"cwe_claimed_twice", "cwe_claimed_twice",
"mechanism_claimed_twice",
"no_version", "no_version",
"unknown_parent", "unknown_parent",
], ],
+3 -1
View File
@@ -46,4 +46,6 @@ def test_every_vendored_typescript_pair_parses_on_both_sides() -> None:
elif not ir.parse_ok: elif not ir.parse_ok:
failed.append(f"{case.name}:{side}") failed.append(f"{case.name}:{side}")
assert unsupported == [] assert unsupported == []
assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar # One excerpt is a truncated destructuring parameter list, so it is not valid TypeScript on its own;
# that is a corpus defect for the re-harvest, not a grammar gap. Pinned exactly so a regression shows.
assert failed == ["lensops-kubernetesmanagement-fe4910:vuln", "lensops-kubernetesmanagement-fe4910:fixed"], failed