mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
docs(learning-harness): amend the taxonomy name and carry the flat-taxonomy caveat
The maintainer settled both items the group 1 review escalated. The family that covers server-side request forgery is named untrusted destination and also covers open redirect, because both are untrusted input choosing where a request goes and the corpus carries 34 open-redirect rows that would otherwise be unclassifiable. The requirement, the brief and the design example now say so, and the approval note records the amendment. The shipped taxonomy is flat, so no answer can earn hierarchical partial credit. Rather than leave that as prose, two new acceptance criteria require every agreement figure and every family figure to say it, and both report types now carry the flag, computed from the taxonomy rather than asserted. A taxonomy that later declares a parent flips it without any other change. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
86ed54c2e4
commit
e44d45f488
@@ -262,6 +262,7 @@ class ClassifierReport:
|
||||
per_family: dict[str, int] = field(default_factory=dict)
|
||||
unknown: int = 0
|
||||
queue: tuple[QueueEntry, ...] = ()
|
||||
hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 2.6)
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
@@ -275,6 +276,7 @@ class ClassifierReport:
|
||||
"per_tier": dict(sorted(self.per_tier.items())),
|
||||
"per_family": dict(sorted(self.per_family.items())),
|
||||
"unknown": self.unknown,
|
||||
"hierarchical_credit": self.hierarchical_credit,
|
||||
"queue": [entry.to_dict() for entry in self.queue],
|
||||
}
|
||||
|
||||
@@ -334,6 +336,7 @@ def measure_classifier(
|
||||
per_family=per_family,
|
||||
unknown=unknown,
|
||||
queue=tuple(queue),
|
||||
hierarchical_credit=taxonomy.hierarchical,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -68,6 +68,11 @@ class FamilyTaxonomy:
|
||||
def family_of_mechanism(self, mechanism: str) -> Family | None:
|
||||
return next((family for family in self.families if mechanism in family.mechanisms), None)
|
||||
|
||||
@property
|
||||
def hierarchical(self) -> bool:
|
||||
"""True when some family declares a parent. While this is False, partial credit can never be earned."""
|
||||
return any(family.parent for family in self.families)
|
||||
|
||||
def related(self, left: str, right: str) -> Relation:
|
||||
"""How two family answers relate. Abstention is not partial credit: ``unknown`` against a real
|
||||
family is ``lateral``, never ``parent_child``."""
|
||||
|
||||
@@ -181,6 +181,7 @@ class FamilyMetrics:
|
||||
positive_flips: int = 0
|
||||
p_value: float | None = None # two-sided sign test over the discordant pairs
|
||||
reliable_change: bool = False
|
||||
hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 3.7)
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
@@ -199,6 +200,7 @@ class FamilyMetrics:
|
||||
"positive_flips": self.positive_flips,
|
||||
"p_value": self.p_value,
|
||||
"reliable_change": self.reliable_change,
|
||||
"hierarchical_credit": self.hierarchical_credit,
|
||||
}
|
||||
|
||||
|
||||
@@ -258,6 +260,7 @@ def aggregate(
|
||||
positive_flips=positive,
|
||||
p_value=p_value,
|
||||
reliable_change=bool(p_value is not None and p_value < alpha and positive != negative),
|
||||
hierarchical_credit=taxonomy.hierarchical,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
Reference in New Issue
Block a user