mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Fable 5.1
537b61e8d9
Two instrument findings from the first access-control feature table (952 rows, 74 positive rows): 1. A fix that adds a guard is, replayed base-to-head, a push that removes it. The diff producer only kept hunks with added lines at head, so 38 of 74 positive rows had no hunk in their file at all and 69 of 74 were "not on a changed line", while 60% of ordinary rows were. The diff annotation separated the labels by construction and was blind to exactly the access-control shape. The producer (version 2) now records per file the head lines after which the push removed code; distance and on_changed_line count them, the model's hunks input renders them as removed_after_line, and diff.deletion_count is an engineered feature. 2. 150 rows (16%, every positive of one repository) read empty files from the worktree although git holds the blobs; the build ran while another process checked out from the same clone. _full_checkout now proves the checkout: git status must be clean, a dirty tree is restored once, then InstrumentFailure checkout_incomplete. The table script cross-checks every unread row against the blob size and refuses to screen a partly unread tree, and records the diff-scoped recall ceiling (positive rows a label-blind enumeration would reach). Co-Authored-By: Claude Fable 5.1 <[email protected]>
103 lines
4.3 KiB
Python
103 lines
4.3 KiB
Python
"""Engineered features from the feature table, their tie to the annotations, and the separation screen."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
pd = pytest.importorskip("pandas")
|
|
pytest.importorskip("sklearn")
|
|
|
|
from openultrasast.hunt import features # noqa: E402
|
|
|
|
ROW = {
|
|
"change_id": "c1",
|
|
"repo": "r",
|
|
"head": "h",
|
|
"path": "app.py",
|
|
"line": 5,
|
|
"kind": "vulnerability",
|
|
"fold": 1,
|
|
"label": 1,
|
|
"family": "injection",
|
|
"feature.diff.hunks": [[3, 3], [10, 12]],
|
|
"feature.diff.deletions": [9],
|
|
"feature.diff.on_changed_line": False,
|
|
"feature.diff.distance": 2,
|
|
"feature.source.function": "second",
|
|
"feature.source.span": [5, 7],
|
|
"feature.source.callers": [["web.py", 2], ["web.py", 9], ["cli.py", 1]],
|
|
"feature.source.entry_points": [{"path": "web.py", "line": 1, "kind": "route"}],
|
|
"feature.source.language": "python",
|
|
"feature.semantic.roles": {"input": "source", "eval": "sink", "escape": "sanitizer", "x": "sink"},
|
|
"feature.semantic.def_use": {"binds": [{"line": 6}], "calls": [{"line": 6}, {"line": 7}]},
|
|
"feature.semantic.obligations": [
|
|
{"operation": "auth", "line": 6, "discharged": False},
|
|
{"operation": "csrf", "line": 7, "discharged": True},
|
|
],
|
|
"feature.taxonomy.families": [{"id": "injection", "definition": "x"}],
|
|
"provenance.diff.ok": True,
|
|
"provenance.source.ok": True,
|
|
"provenance.semantic.ok": True,
|
|
"provenance.taxonomy.ok": True,
|
|
}
|
|
|
|
|
|
def test_feature_frame_engineers_counts_distances_and_flags():
|
|
absent = {
|
|
**ROW,
|
|
"change_id": "c2",
|
|
"label": 0,
|
|
"feature.source.function": "<module>",
|
|
"feature.source.span": None,
|
|
"feature.semantic.roles": None,
|
|
"feature.semantic.def_use": None,
|
|
"feature.semantic.obligations": None,
|
|
"provenance.semantic.ok": False,
|
|
}
|
|
frame = features.feature_frame([ROW, absent])
|
|
first = frame.iloc[0]
|
|
assert first["diff.distance"] == 2 and first["diff.on_changed_line"] == 0
|
|
assert first["diff.hunk_count"] == 2 and first["diff.hunk_lines"] == 4 and first["diff.deletion_count"] == 1
|
|
assert first["source.span_lines"] == 3 and first["source.is_module"] == 0
|
|
assert first["source.caller_count"] == 3 and first["source.caller_files"] == 2 and first["source.entry_point_count"] == 1
|
|
assert (first["semantic.sources"], first["semantic.sinks"], first["semantic.sanitizers"]) == (1, 2, 1)
|
|
assert first["semantic.binds"] == 1 and first["semantic.calls"] == 2
|
|
assert first["semantic.obligations"] == 2 and first["semantic.obligations_undischarged"] == 1
|
|
second = frame.iloc[1]
|
|
assert second["source.is_module"] == 1 and pd.isna(second["source.span_lines"])
|
|
assert pd.isna(second["semantic.sinks"]) and second["ok.semantic"] is False or second["ok.semantic"] == False # noqa: E712
|
|
assert list(frame["label"]) == [1, 0]
|
|
|
|
|
|
def test_every_engineered_feature_maps_to_an_annotation_and_masks_follow():
|
|
from openultrasast.hunt.program import INPUTS
|
|
|
|
inputs = {name for name, _, _ in INPUTS}
|
|
assert all(annotation in inputs for _, annotation in features.ENGINEERED.values())
|
|
assert set(features.features_of("roles")) == {"semantic.sources", "semantic.sinks", "semantic.sanitizers"}
|
|
assert features.annotations_of(["semantic"]) == {"roles", "def_use", "obligations"}
|
|
assert features.annotations_of(["feature.source.callers"]) == {"callers"}
|
|
assert features.annotations_of(["hunks"]) == {"hunks"}
|
|
assert features.annotations_of([]) == set()
|
|
|
|
|
|
def test_screen_reports_the_strongest_feature_per_annotation():
|
|
rows = []
|
|
for i in range(12):
|
|
label = i % 2
|
|
rows.append(
|
|
{
|
|
**ROW,
|
|
"change_id": f"c{i}",
|
|
"label": label,
|
|
"feature.diff.distance": 0 if label else 30,
|
|
"feature.source.callers": [["a.py", 1]] * (3 if label else 1),
|
|
"feature.semantic.roles": {"eval": "sink"},
|
|
}
|
|
)
|
|
frame = features.feature_frame(rows)
|
|
out = features.screen(frame).set_index("annotation")
|
|
assert out.loc["hunks", "strongest_feature"] == "diff.distance" and out.loc["hunks", "auc"] == 0.0
|
|
assert out.loc["callers", "strongest_feature"] == "source.caller_count" and out.loc["callers", "auc"] == 1.0
|
|
assert out.loc["roles", "auc"] == 0.5 # constant: no signal
|