mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 20:19:29 +00:00
Marius CiepluchandClaude Opus 5.5
50c93edfb8
Real (vulnerable function, fixed function) pointer pairs for the six compiled families, mined at zero model cost from reviewed GHSA advisories (pip, npm, maven, composer), newest first: 79 of 98 published after 2026-06-01 (post_cutoff = true). Each pair: one linked one-parent fix commit, <= 4 source files / 120 lines, the function holding the changed code lines declared once on both sides with a changed body; every function diff read and kept only when the named function holds the fix (145 reviewed, 100 kept); 2 dropped by the reserved-population test; permissive licences only, no repository named elsewhere under benchmarks/. New label source advisory-fixes-2; the advisory-fix tests run over both batches. Memory 937 -> 1133 static examples (98 paired groups added); positives-only groups stay 99 (vibe-py/dev-php/v2 data gaps), their share of positive groups in the six families 0.40 -> 0.29. Embeddings: 7 calls, $0.0014. No chat-model call. Record: benchmarks/measurements/2026-10-03-decision-engine-negatives-harvest. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
137 lines
5.6 KiB
TOML
137 lines
5.6 KiB
TOML
# Label sources, fail-closed (learned-decision-engine Req 2.3; design section 3).
|
|
#
|
|
# `ousast learn labels` reads ONLY the files named here. A population that is not listed cannot produce a label, and
|
|
# the builder never opens a file it does not list: every read goes through one guard that refuses any path outside
|
|
# this list and every `[[excluded]]` file name, before the file is touched.
|
|
#
|
|
# Population v3 (PHP) is reserved for the one-time check (Req 5.3, design section 7). It is named below only as
|
|
# excluded, by file name; nothing in it is read. It becomes a label source only in a later commit that adds it here
|
|
# after `result-v3-engine.json` is recorded.
|
|
#
|
|
# Paths are relative to the repository root. `evaluation` names the files that prove the source's evaluation was
|
|
# recorded (checked to exist, never parsed for labels); `recorded` is that record's commit date.
|
|
|
|
version = 1
|
|
|
|
[[source]]
|
|
id = "population-v1"
|
|
kind = "population"
|
|
file = "benchmarks/independent/population-v1.toml"
|
|
evaluation = ["benchmarks/independent/results-v1.json"]
|
|
recorded = "2026-09-25"
|
|
status = "spent" # retuned on after its evaluation (population-v2.toml header)
|
|
matching = "fix_range" # v1 declares no sites: the functions the fix changed on the vulnerable side
|
|
cache = "independent" # one clone per case id under ~/.cache/openultrasast/independent
|
|
|
|
[[source]]
|
|
id = "population-v2"
|
|
kind = "population"
|
|
file = "benchmarks/independent/population-v2.toml"
|
|
evaluation = [
|
|
"benchmarks/independent/results-v2.json",
|
|
"benchmarks/independent/results-v2-hunter.json",
|
|
"benchmarks/independent/plane-increment-2.json",
|
|
]
|
|
recorded = "2026-09-28"
|
|
status = "spent"
|
|
matching = "declared_sites"
|
|
design_informed = true # verify, the tie-break and the callers line were chosen after seeing v2 (design section 4)
|
|
cache = "independent"
|
|
|
|
[[source]]
|
|
id = "dev-php"
|
|
kind = "recipes"
|
|
files = ["benchmarks/repos/pmpro.toml", "benchmarks/repos/wpstatistics.toml", "benchmarks/repos/mwwpform.toml"]
|
|
evaluation = ["benchmarks/push/inputs.json"]
|
|
recorded = "2026-09-12"
|
|
status = "development"
|
|
checkouts = "repos" # ~/.cache/openultrasast/repos/<name>, read for dependency manifests only
|
|
|
|
[[source]]
|
|
id = "pairs"
|
|
kind = "pairs"
|
|
file = "benchmarks/pairs/catalog.toml" # with the slice catalogs load_pair_catalog adds
|
|
evaluation = ["benchmarks/measurements/2026-09-30-harnessx-removal-baseline/pairs.json"]
|
|
recorded = "2026-09-30"
|
|
status = "development"
|
|
exclude_tiers = ["title"] # labels read from advisory titles; a sensitivity arm only (`--include-title`)
|
|
|
|
# Real fix-commit pairs for the families the pair corpus left without negatives (untrusted_destination, path,
|
|
# output_encoding, deserialization, access_control). Each pair is a public advisory fix commit distinct from its parent;
|
|
# the advisory-named function is declared on both sides and its body changes. Pointer pairs: both sides are
|
|
# materialised into the pair cache by benchmarks/pairs/advisory-fixes/materialize.py (shallow blobless fetch, one
|
|
# file per side); `evaluation` is the committed per-pair verification record (bytes read, declared both sides).
|
|
[[source]]
|
|
id = "advisory-fixes"
|
|
kind = "pairs"
|
|
file = "benchmarks/pairs/advisory-fixes/catalog.toml"
|
|
evaluation = ["benchmarks/pairs/advisory-fixes/verification.json"]
|
|
recorded = "2026-10-01"
|
|
status = "development"
|
|
exclude_tiers = ["title"]
|
|
|
|
# Batch 2: advisory fixes whose changed function survives on the fixed side, for the six compiled families (most
|
|
# published after 2026-06-01). Same slice, cache layout and checks as batch 1; screening record
|
|
# benchmarks/measurements/2026-10-03-decision-engine-negatives-harvest/record.json.
|
|
[[source]]
|
|
id = "advisory-fixes-2"
|
|
kind = "pairs"
|
|
file = "benchmarks/pairs/advisory-fixes-2/catalog.toml"
|
|
evaluation = ["benchmarks/pairs/advisory-fixes-2/verification.json"]
|
|
recorded = "2026-10-03"
|
|
status = "development"
|
|
exclude_tiers = ["title"]
|
|
|
|
[[source]]
|
|
id = "adjudications"
|
|
kind = "adjudications"
|
|
files = [
|
|
"benchmarks/independent/results-v1.json#precision_review",
|
|
"benchmarks/independent/results-v2.json#precision_sample",
|
|
"benchmarks/independent/results-v2-hunter.json#precision_sample",
|
|
]
|
|
evaluation = ["benchmarks/independent/results-v1.json", "benchmarks/independent/results-v2.json"]
|
|
recorded = "2026-09-28"
|
|
status = "recorded verdicts"
|
|
# verifier-check-2026-09-29.json records aggregate counts only (no per-finding verdict), so it labels nothing.
|
|
|
|
# Maintainer decision at design approval (2026-09-30): ordinary non-security commits of the training repositories are
|
|
# mined as ASSUMED-BENIGN pushes -- their own source, reported separately, never merged with verified negatives.
|
|
[[source]]
|
|
id = "assumed_benign"
|
|
kind = "history"
|
|
repositories_from = ["population-v1", "population-v2"]
|
|
recorded = "2026-09-30"
|
|
status = "assumption"
|
|
weight = 0.5
|
|
min_days_from_fix = 90
|
|
scan_commits = 1500 # newest ancestors of the fixed pin looked at, per repository
|
|
max_commits_per_repository = 40
|
|
max_files_per_commit = 20 # a larger commit is a bulk change, not a push
|
|
|
|
[[excluded]]
|
|
file = "population-v3-php.toml"
|
|
reason = "population v3 is reserved for the one-time check (protocol-v3.md); never opened"
|
|
|
|
[[excluded]]
|
|
file = "freeze-v3-php.json"
|
|
reason = "v3 freeze record; never opened"
|
|
|
|
[[excluded]]
|
|
file = "candidates-v3-php.json"
|
|
reason = "v3 candidate list; never opened"
|
|
|
|
[[excluded]]
|
|
file = "research-2026-09-30-php.md"
|
|
reason = "v3 selection research; never opened"
|
|
|
|
[[excluded]]
|
|
file = "protocol-v3.md"
|
|
reason = "v3 protocol; never opened"
|
|
|
|
# Repository aliases: forks and renames that are one repository group. Groups sharing an advisory id (CVE/GHSA) or
|
|
# a fix commit are merged automatically; list here only what those cannot see.
|
|
# [[alias]]
|
|
# repo = "someone/fork"
|
|
# group = "upstream/project"
|