docs(learning-harness): amend the taxonomy name and carry the flat-taxonomy caveat

The maintainer settled both items the group 1 review escalated.

The family that covers server-side request forgery is named untrusted
destination and also covers open redirect, because both are untrusted input
choosing where a request goes and the corpus carries 34 open-redirect rows that
would otherwise be unclassifiable. The requirement, the brief and the design
example now say so, and the approval note records the amendment.

The shipped taxonomy is flat, so no answer can earn hierarchical partial credit.
Rather than leave that as prose, two new acceptance criteria require every
agreement figure and every family figure to say it, and both report types now
carry the flag, computed from the taxonomy rather than asserted. A taxonomy that
later declares a parent flips it without any other change.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-06 10:22:33 +02:00
co-authored by Claude Opus 5
parent 86ed54c2e4
commit e44d45f488
9 changed files with 46 additions and 5 deletions
+3
View File
@@ -262,6 +262,7 @@ class ClassifierReport:
per_family: dict[str, int] = field(default_factory=dict)
unknown: int = 0
queue: tuple[QueueEntry, ...] = ()
hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 2.6)
def to_dict(self) -> dict[str, object]:
return {
@@ -275,6 +276,7 @@ class ClassifierReport:
"per_tier": dict(sorted(self.per_tier.items())),
"per_family": dict(sorted(self.per_family.items())),
"unknown": self.unknown,
"hierarchical_credit": self.hierarchical_credit,
"queue": [entry.to_dict() for entry in self.queue],
}
@@ -334,6 +336,7 @@ def measure_classifier(
per_family=per_family,
unknown=unknown,
queue=tuple(queue),
hierarchical_credit=taxonomy.hierarchical,
)
+5
View File
@@ -68,6 +68,11 @@ class FamilyTaxonomy:
def family_of_mechanism(self, mechanism: str) -> Family | None:
return next((family for family in self.families if mechanism in family.mechanisms), None)
@property
def hierarchical(self) -> bool:
"""True when some family declares a parent. While this is False, partial credit can never be earned."""
return any(family.parent for family in self.families)
def related(self, left: str, right: str) -> Relation:
"""How two family answers relate. Abstention is not partial credit: ``unknown`` against a real
family is ``lateral``, never ``parent_child``."""
+3
View File
@@ -181,6 +181,7 @@ class FamilyMetrics:
positive_flips: int = 0
p_value: float | None = None # two-sided sign test over the discordant pairs
reliable_change: bool = False
hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 3.7)
def to_dict(self) -> dict[str, object]:
return {
@@ -199,6 +200,7 @@ class FamilyMetrics:
"positive_flips": self.positive_flips,
"p_value": self.p_value,
"reliable_change": self.reliable_change,
"hierarchical_credit": self.hierarchical_credit,
}
@@ -258,6 +260,7 @@ def aggregate(
positive_flips=positive,
p_value=p_value,
reliable_change=bool(p_value is not None and p_value < alpha and positive != negative),
hierarchical_credit=taxonomy.hierarchical,
)
return out