docs(learning-harness): amend the taxonomy name and carry the flat-taxonomy caveat

The maintainer settled both items the group 1 review escalated.

The family that covers server-side request forgery is named untrusted
destination and also covers open redirect, because both are untrusted input
choosing where a request goes and the corpus carries 34 open-redirect rows that
would otherwise be unclassifiable. The requirement, the brief and the design
example now say so, and the approval note records the amendment.

The shipped taxonomy is flat, so no answer can earn hierarchical partial credit.
Rather than leave that as prose, two new acceptance criteria require every
agreement figure and every family figure to say it, and both report types now
carry the flag, computed from the taxonomy rather than asserted. A taxonomy that
later declares a parent flips it without any other change.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 10:22:33 +02:00
co-authored by Claude Opus 5
parent 86ed54c2e4
commit e44d45f488
9 changed files with 46 additions and 5 deletions
+18
View File
@@ -96,3 +96,21 @@ def test_the_sign_test_is_two_sided_and_symmetric() -> None:
assert sign_test(0, 0) == 1.0
assert sign_test(6, 0) == sign_test(0, 6) < 0.05
assert sign_test(1, 0) > 0.05 and sign_test(3, 3) == 1.0
def test_every_family_figure_says_when_hierarchical_credit_could_not_apply() -> None:
"""Req 3.7: with a flat taxonomy no answer can earn partial credit, and the number must carry that."""
from openultrasast.learning.families import Family, FamilyTaxonomy
from openultrasast.learning.scoring import aggregate
flat = load_families()
metrics = aggregate([_score("a", "pair_correct")], taxonomy=flat)["access_control"]
assert metrics.hierarchical_credit is False and metrics.to_dict()["hierarchical_credit"] is False
nested = FamilyTaxonomy(
version="test",
families=(
Family(id="injection", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none"),
Family(id="access_control", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none", parent="injection"),
),
)
assert aggregate([_score("a", "pair_correct")], taxonomy=nested)["access_control"].hierarchical_credit is True
+10
View File
@@ -99,3 +99,13 @@ def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -
report = measure_classifier(cases, taxonomy, client=None)
assert report.labeled == 1 and report.classified == 2 # an unreviewed label is not a reference answer
assert [entry.pair for entry in report.queue] == ["trusted"]
def test_the_report_says_when_hierarchical_credit_could_not_apply(tmp_path: Path) -> None:
"""Req 2.6: the shipped taxonomy is flat, so partial credit is inert and every figure must say so."""
from openultrasast.learning.classify import measure_classifier
taxonomy = load_families()
report = measure_classifier([_case(tmp_path, "a", cwe="CWE-89", family="injection")], taxonomy, client=None)
assert report.hierarchical_credit is False
assert report.to_dict()["hierarchical_credit"] is False