mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
test(learning-harness): apply the group 1 review suggestions
A mechanism claimed by two families now fails as loudly as a weakness claimed twice, with a named offender. Two behaviours that were correct but unasserted are pinned: the obligations tool refuses a path outside the repository, and a provider that refuses to build degrades to no endpoint instead of raising into a scan. The TypeScript parse assertion is now exact rather than a quarter-of- the-slice allowance, which surfaces one real corpus defect: a TSX excerpt harvested as a truncated destructuring parameter list is not valid TypeScript on its own, and belongs to the re-harvest rather than to the grammar. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
d05c516a10
commit
17fc18c785
@@ -99,6 +99,7 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
|
|||||||
raise FamiliesError(f"{source}: version is required")
|
raise FamiliesError(f"{source}: version is required")
|
||||||
families: list[Family] = []
|
families: list[Family] = []
|
||||||
claimed: dict[str, str] = {}
|
claimed: dict[str, str] = {}
|
||||||
|
claimed_mechanisms: dict[str, str] = {}
|
||||||
for entry in payload.get("family", []):
|
for entry in payload.get("family", []):
|
||||||
if not isinstance(entry, dict):
|
if not isinstance(entry, dict):
|
||||||
raise FamiliesError(f"{source}: every [[family]] entry must be a table")
|
raise FamiliesError(f"{source}: every [[family]] entry must be a table")
|
||||||
@@ -109,6 +110,12 @@ def load_families(path: Path | None = None, *, require_all: bool = True) -> Fami
|
|||||||
if cwe in claimed:
|
if cwe in claimed:
|
||||||
raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}")
|
raise FamiliesError(f"{source}: {cwe} is claimed by both {claimed[cwe]!r} and {family.id!r}")
|
||||||
claimed[cwe] = family.id
|
claimed[cwe] = family.id
|
||||||
|
for mechanism in sorted(family.mechanisms):
|
||||||
|
if mechanism in claimed_mechanisms:
|
||||||
|
raise FamiliesError(
|
||||||
|
f"{source}: mechanism {mechanism} is claimed by both {claimed_mechanisms[mechanism]!r} and {family.id!r}"
|
||||||
|
)
|
||||||
|
claimed_mechanisms[mechanism] = family.id
|
||||||
families.append(family)
|
families.append(family)
|
||||||
_check_parents(source, families)
|
_check_parents(source, families)
|
||||||
if require_all and tuple(family.id for family in families) != FAMILY_IDS:
|
if require_all and tuple(family.id for family in families) != FAMILY_IDS:
|
||||||
|
|||||||
@@ -65,6 +65,8 @@ def test_obligations_lists_what_a_function_owes_without_dumping_code(repo: Path)
|
|||||||
assert "identity_constraint" in {str(row["missing"]) for row in rows}
|
assert "identity_constraint" in {str(row["missing"]) for row in rows}
|
||||||
assert all("text" not in row and "content" not in row for row in rows)
|
assert all("text" not in row and "content" not in row for row in rows)
|
||||||
assert obligations(repo, path="app.py", function="helper") == []
|
assert obligations(repo, path="app.py", function="helper") == []
|
||||||
|
with pytest.raises(PathEscapesRepo):
|
||||||
|
obligations(repo, path="../outside.py", function="leaky")
|
||||||
|
|
||||||
|
|
||||||
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
|
def test_flows_lists_adjudicated_source_to_sink_records_for_one_function(repo: Path) -> None:
|
||||||
|
|||||||
@@ -67,6 +67,18 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke
|
|||||||
assert resolve_chat_endpoint(ResolvedConfig()) is None
|
assert resolve_chat_endpoint(ResolvedConfig()) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from openultrasast.learning.endpoint import resolve_chat_endpoint
|
||||||
|
|
||||||
|
_endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key")
|
||||||
|
|
||||||
|
def _refuse() -> provider.OpenRouterChatClient:
|
||||||
|
raise provider.OpenRouterError("no key after all")
|
||||||
|
|
||||||
|
monkeypatch.setattr(provider.OpenRouterChatClient, "from_env", staticmethod(_refuse))
|
||||||
|
assert resolve_chat_endpoint(ResolvedConfig()) is None # a caller degrades; nothing raises into a scan
|
||||||
|
|
||||||
|
|
||||||
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||||
from openultrasast.config import load_config
|
from openultrasast.config import load_config
|
||||||
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
|
from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models
|
||||||
|
|||||||
@@ -88,6 +88,10 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
|
|||||||
(GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"),
|
(GOOD.replace('cwes = ["CWE-89"]', 'cwes = ["CWE-89", "CWE-89"]'), "CWE-89"),
|
||||||
(GOOD.replace('description = "x"\n', ""), "description"),
|
(GOOD.replace('description = "x"\n', ""), "description"),
|
||||||
(GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"),
|
(GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = ["CWE-89"]\nmechanisms = []\nverifier = "canary"\n', "CWE-89"),
|
||||||
|
(
|
||||||
|
GOOD + '\n[[family]]\nid = "path"\ndescription = "y"\ncwes = []\nmechanisms = ["source_reaches_sink"]\nverifier = "canary"\n',
|
||||||
|
"source_reaches_sink",
|
||||||
|
),
|
||||||
(GOOD.replace('version = "1"\n', ""), "version"),
|
(GOOD.replace('version = "1"\n', ""), "version"),
|
||||||
(GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"),
|
(GOOD + '\n[[family]]\nid = "path"\nparent = "nope"\ndescription = "y"\ncwes = []\nmechanisms = []\nverifier = "none"\n', "nope"),
|
||||||
],
|
],
|
||||||
@@ -99,6 +103,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared
|
|||||||
"duplicate_cwe",
|
"duplicate_cwe",
|
||||||
"no_description",
|
"no_description",
|
||||||
"cwe_claimed_twice",
|
"cwe_claimed_twice",
|
||||||
|
"mechanism_claimed_twice",
|
||||||
"no_version",
|
"no_version",
|
||||||
"unknown_parent",
|
"unknown_parent",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -46,4 +46,6 @@ def test_every_vendored_typescript_pair_parses_on_both_sides() -> None:
|
|||||||
elif not ir.parse_ok:
|
elif not ir.parse_ok:
|
||||||
failed.append(f"{case.name}:{side}")
|
failed.append(f"{case.name}:{side}")
|
||||||
assert unsupported == []
|
assert unsupported == []
|
||||||
assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar
|
# One excerpt is a truncated destructuring parameter list, so it is not valid TypeScript on its own;
|
||||||
|
# that is a corpus defect for the re-harvest, not a grammar gap. Pinned exactly so a regression shows.
|
||||||
|
assert failed == ["lensops-kubernetesmanagement-fe4910:vuln", "lensops-kubernetesmanagement-fe4910:fixed"], failed
|
||||||
|
|||||||
Reference in New Issue
Block a user