mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 13:10:03 +00:00
feat(learn): labels from ground truth with provenance (decision-engine task 4)
learn/sources.toml is the fail-closed source list: every read goes through one guard that refuses an unlisted path or an [[excluded]] file name before opening it, and a source whose evaluation record is missing cannot label. Population v3 is named only as excluded, by file name, and is never opened (tested with a v3-named fixture and open/read_text tracked). learn/labels.py and `ousast learn labels` apply the design's rules: v2 declared sites and v1 fix-range functions positive, the same function on the fixed pin negative only when the fix changed it, introduce/repair delta units, benign control pushes as conditional negatives; pairs vulnerable/fixed (unscorable and title tier out, title as a sensitivity arm); dev-php recipe known functions; recorded adjudications with their reasons. v2 rows are labels marked design_informed, as the design says. Repository groups: normalised owner/name, merged by shared CVE/GHSA, shared fix commits and [[alias]] rows. assumed_benign (maintainer decision): ordinary commits of the population repositories, >= 90 days from any fix, no later security commit touching their files, weight 0.5, conditional, reported separately and never merged with verified negatives. The security filter is multilingual (CJK, Korean, Russian, European and other languages) plus CVE/GHSA/CWE references and dependency bumps; widened after reading accepted pushes (validation, passwords, guards). First snapshot, counts only (benchmarks/measurements/2026-09-30-decision-engine- labels/labels.json): verified pin labels in positive/negative repository groups injection 44/41, untrusted_destination 35/34, deserialization 20/20 (per-family), output_encoding 15/15, path 15/14, access_control 12/12 (pooled), config_secrets 5/5, memory 5/5 (insufficient). assumed_benign: 969 pushes in 27 groups from 36,000 commits scanned. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
884092e62c
commit
e6ed7854bb
@@ -0,0 +1,110 @@
|
||||
# Label sources, fail-closed (learned-decision-engine Req 2.3; design section 3).
|
||||
#
|
||||
# `ousast learn labels` reads ONLY the files named here. A population that is not listed cannot produce a label, and
|
||||
# the builder never opens a file it does not list: every read goes through one guard that refuses any path outside
|
||||
# this list and every `[[excluded]]` file name, before the file is touched.
|
||||
#
|
||||
# Population v3 (PHP) is reserved for the one-time check (Req 5.3, design section 7). It is named below only as
|
||||
# excluded, by file name; nothing in it is read. It becomes a label source only in a later commit that adds it here
|
||||
# after `result-v3-engine.json` is recorded.
|
||||
#
|
||||
# Paths are relative to the repository root. `evaluation` names the files that prove the source's evaluation was
|
||||
# recorded (checked to exist, never parsed for labels); `recorded` is that record's commit date.
|
||||
|
||||
version = 1
|
||||
|
||||
[[source]]
|
||||
id = "population-v1"
|
||||
kind = "population"
|
||||
file = "benchmarks/independent/population-v1.toml"
|
||||
evaluation = ["benchmarks/independent/results-v1.json"]
|
||||
recorded = "2026-09-25"
|
||||
status = "spent" # retuned on after its evaluation (population-v2.toml header)
|
||||
matching = "fix_range" # v1 declares no sites: the functions the fix changed on the vulnerable side
|
||||
cache = "independent" # one clone per case id under ~/.cache/openultrasast/independent
|
||||
|
||||
[[source]]
|
||||
id = "population-v2"
|
||||
kind = "population"
|
||||
file = "benchmarks/independent/population-v2.toml"
|
||||
evaluation = [
|
||||
"benchmarks/independent/results-v2.json",
|
||||
"benchmarks/independent/results-v2-hunter.json",
|
||||
"benchmarks/independent/plane-increment-2.json",
|
||||
]
|
||||
recorded = "2026-09-28"
|
||||
status = "spent"
|
||||
matching = "declared_sites"
|
||||
design_informed = true # verify, the tie-break and the callers line were chosen after seeing v2 (design section 4)
|
||||
cache = "independent"
|
||||
|
||||
[[source]]
|
||||
id = "dev-php"
|
||||
kind = "recipes"
|
||||
files = ["benchmarks/repos/pmpro.toml", "benchmarks/repos/wpstatistics.toml", "benchmarks/repos/mwwpform.toml"]
|
||||
evaluation = ["benchmarks/push/inputs.json"]
|
||||
recorded = "2026-09-12"
|
||||
status = "development"
|
||||
checkouts = "repos" # ~/.cache/openultrasast/repos/<name>, read for dependency manifests only
|
||||
|
||||
[[source]]
|
||||
id = "pairs"
|
||||
kind = "pairs"
|
||||
file = "benchmarks/pairs/catalog.toml" # with the slice catalogs load_pair_catalog adds
|
||||
evaluation = ["benchmarks/measurements/2026-09-30-harnessx-removal-baseline/pairs.json"]
|
||||
recorded = "2026-09-30"
|
||||
status = "development"
|
||||
exclude_tiers = ["title"] # labels read from advisory titles; a sensitivity arm only (`--include-title`)
|
||||
|
||||
[[source]]
|
||||
id = "adjudications"
|
||||
kind = "adjudications"
|
||||
files = [
|
||||
"benchmarks/independent/results-v1.json#precision_review",
|
||||
"benchmarks/independent/results-v2.json#precision_sample",
|
||||
"benchmarks/independent/results-v2-hunter.json#precision_sample",
|
||||
]
|
||||
evaluation = ["benchmarks/independent/results-v1.json", "benchmarks/independent/results-v2.json"]
|
||||
recorded = "2026-09-28"
|
||||
status = "recorded verdicts"
|
||||
# verifier-check-2026-09-29.json records aggregate counts only (no per-finding verdict), so it labels nothing.
|
||||
|
||||
# Maintainer decision at design approval (2026-09-30): ordinary non-security commits of the training repositories are
|
||||
# mined as ASSUMED-BENIGN pushes -- their own source, reported separately, never merged with verified negatives.
|
||||
[[source]]
|
||||
id = "assumed_benign"
|
||||
kind = "history"
|
||||
repositories_from = ["population-v1", "population-v2"]
|
||||
recorded = "2026-09-30"
|
||||
status = "assumption"
|
||||
weight = 0.5
|
||||
min_days_from_fix = 90
|
||||
scan_commits = 1500 # newest ancestors of the fixed pin looked at, per repository
|
||||
max_commits_per_repository = 40
|
||||
max_files_per_commit = 20 # a larger commit is a bulk change, not a push
|
||||
|
||||
[[excluded]]
|
||||
file = "population-v3-php.toml"
|
||||
reason = "population v3 is reserved for the one-time check (protocol-v3.md); never opened"
|
||||
|
||||
[[excluded]]
|
||||
file = "freeze-v3-php.json"
|
||||
reason = "v3 freeze record; never opened"
|
||||
|
||||
[[excluded]]
|
||||
file = "candidates-v3-php.json"
|
||||
reason = "v3 candidate list; never opened"
|
||||
|
||||
[[excluded]]
|
||||
file = "research-2026-09-30-php.md"
|
||||
reason = "v3 selection research; never opened"
|
||||
|
||||
[[excluded]]
|
||||
file = "protocol-v3.md"
|
||||
reason = "v3 protocol; never opened"
|
||||
|
||||
# Repository aliases: forks and renames that are one repository group. Groups sharing an advisory id (CVE/GHSA) or
|
||||
# a fix commit are merged automatically; list here only what those cannot see.
|
||||
# [[alias]]
|
||||
# repo = "someone/fork"
|
||||
# group = "upstream/project"
|
||||
Reference in New Issue
Block a user