mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
learn: 98 advisory-fix pairs whose function survives the fix (advisory-fixes-2)
Real (vulnerable function, fixed function) pointer pairs for the six compiled families, mined at zero model cost from reviewed GHSA advisories (pip, npm, maven, composer), newest first: 79 of 98 published after 2026-06-01 (post_cutoff = true). Each pair: one linked one-parent fix commit, <= 4 source files / 120 lines, the function holding the changed code lines declared once on both sides with a changed body; every function diff read and kept only when the named function holds the fix (145 reviewed, 100 kept); 2 dropped by the reserved-population test; permissive licences only, no repository named elsewhere under benchmarks/. New label source advisory-fixes-2; the advisory-fix tests run over both batches. Memory 937 -> 1133 static examples (98 paired groups added); positives-only groups stay 99 (vibe-py/dev-php/v2 data gaps), their share of positive groups in the six families 0.40 -> 0.29. Embeddings: 7 calls, $0.0014. No chat-model call. Record: benchmarks/measurements/2026-10-03-decision-engine-negatives-harvest. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
432fd2947e
commit
50c93edfb8
@@ -0,0 +1,351 @@
|
||||
{
|
||||
"created": "2026-10-03",
|
||||
"note": "COUNTS ONLY. Real (vulnerable function, fixed function) pairs where the function survives the fix, mined at zero model cost from reviewed GitHub advisories for the six compiled families, added as pointer-pair corpus benchmarks/pairs/advisory-fixes-2 (98 pairs, source `advisory-fixes-2` in learn/sources.toml), labelled, given static feature records and built into the memory store; their excerpts embedded. No chat-model call. Candidate, path, function and repository names stay in the catalog; the two pairs the reserved-population test flagged are withheld (never printed: found by bisection with the test's output discarded).",
|
||||
"method": {
|
||||
"enumeration": "gh api /advisories?type=reviewed&ecosystem=<eco>&cwes=<family CWEs>&published=>2025-06-01, per ecosystem (pip, npm, maven, composer, go, rubygems) x family; newest first",
|
||||
"advisories_enumerated": 6594,
|
||||
"advisories_published_after_2026_06_01": 2400,
|
||||
"automated_screen": "single family by CWE; a linked fix commit (at most 3, one repository); repository not named anywhere under benchmarks/ (populations v1/v2, every pair corpus; excluded v3 file names never opened), no WordPress plugin, one pair per repository; SPDX licence permissive; one-parent commit with <= 4 non-test source files and <= 120 changed source lines; the function holding most changed CODE lines (comments and blanks excluded) inside its own matcher span, declared exactly once on both sides, body changed, >= 25% of the file's changed code lines (second pass; the first pass counted lines by nearest declaration above)",
|
||||
"file_reads": "the two versions of each changed source file through the contents API (metadata + two raw files per file; no clone, no scanner, engine, plane task or model on any of these repositories)",
|
||||
"manual_review": "every automated candidate's function diff was read (agent, no model call) and kept only when the named function itself holds the fix",
|
||||
"reserved_population_guard": "tests/test_independent_population.py::test_no_reserved_repository_is_referenced_by_the_development_tree, run with its output discarded; bisection over catalog subsets, 23 runs",
|
||||
"quotas": "20 per family on the second pass (14 on the first); advisories past a met quota were not examined",
|
||||
"mechanism": "family default (injection/output_encoding/deserialization source_reaches_sink, path path_join_user_input, access_control missing_auth_guard, untrusted_destination validation_strength for CWE-601 else source_reaches_sink); provenance = agent"
|
||||
},
|
||||
"screened": {
|
||||
"advisories_examined_second_pass": 2983,
|
||||
"outcomes_second_pass": {
|
||||
"no fix commit linked": 1247,
|
||||
"repository already in benchmarks/ (populations v1/v2 or a pair corpus)": 520,
|
||||
"language outside the feature vocabulary (go/ruby)": 332,
|
||||
"CWEs span several families": 253,
|
||||
"repository already kept (one pair per repository)": 140,
|
||||
"licence not permissive or unrecorded": 135,
|
||||
"large commit (refactor or release, not the fix)": 125,
|
||||
"kept": 110,
|
||||
"no function declared on both sides with a changed body": 51,
|
||||
"fix spread over several commits or repositories": 25,
|
||||
"manual review: the chosen function does not hold the fix (comment-only, misattributed, partial or family mismatch)": 13,
|
||||
"no source file changed": 11,
|
||||
"merge or root commit": 11,
|
||||
"language outside the feature vocabulary (go/ruby/c#/...)": 9,
|
||||
"WordPress plugin (population v3 safety)": 1
|
||||
},
|
||||
"candidates_manually_reviewed": 145,
|
||||
"kept_after_review": 100,
|
||||
"review_drops": {
|
||||
"total": 45,
|
||||
"function misattributed (matcher span overrun or an undetected enclosing declaration)": 13,
|
||||
"change in the function unrelated to the advisory (refactor, rename, formatting, another advisory)": 11,
|
||||
"partial: the function only calls or configures the real fix": 7,
|
||||
"comment or documentation change only": 6,
|
||||
"one pair per repository (a reviewed pair of the same repository kept)": 4,
|
||||
"second pass chose another function of an advisory already reviewed": 2,
|
||||
"family mismatch": 1,
|
||||
"unsure": 1
|
||||
},
|
||||
"reserved_population_drops": 2,
|
||||
"kept": 98
|
||||
},
|
||||
"kept": {
|
||||
"total": 98,
|
||||
"by_family": {
|
||||
"access_control": 17,
|
||||
"deserialization": 11,
|
||||
"injection": 20,
|
||||
"output_encoding": 16,
|
||||
"path": 17,
|
||||
"untrusted_destination": 17
|
||||
},
|
||||
"post_cutoff_by_family": {
|
||||
"access_control": 13,
|
||||
"deserialization": 2,
|
||||
"injection": 20,
|
||||
"output_encoding": 16,
|
||||
"path": 17,
|
||||
"untrusted_destination": 11
|
||||
},
|
||||
"post_cutoff_total": 79,
|
||||
"by_language": {
|
||||
"typescript": 34,
|
||||
"python": 30,
|
||||
"javascript": 15,
|
||||
"java": 10,
|
||||
"php": 9
|
||||
},
|
||||
"by_ecosystem": {
|
||||
"npm": 48,
|
||||
"pip": 32,
|
||||
"maven": 9,
|
||||
"composer": 9
|
||||
},
|
||||
"licences": {
|
||||
"MIT": 67,
|
||||
"Apache-2.0": 24,
|
||||
"BSD-3-Clause": 5,
|
||||
"ISC": 1,
|
||||
"Unlicense": 1
|
||||
},
|
||||
"split": {
|
||||
"train": 51,
|
||||
"holdout": 47
|
||||
},
|
||||
"published_range": [
|
||||
"2025-06-06",
|
||||
"2026-10-02"
|
||||
],
|
||||
"verification": {
|
||||
"verified": 98,
|
||||
"excerpt_bytes_read": 526134
|
||||
},
|
||||
"known_limit": "excerpts inherit the label builder's matcher: some spans run past the function's end into the next declaration; the label is path::function and the changed lines were checked inside the span"
|
||||
},
|
||||
"labels": {
|
||||
"command": "ousast learn labels --root . --out <scratch> --snapshot <scratch>",
|
||||
"rows": [
|
||||
6589,
|
||||
6981
|
||||
],
|
||||
"new_pair_rows": 392,
|
||||
"new_pin_rows_by_label": {
|
||||
"positive": 98,
|
||||
"negative": 98
|
||||
},
|
||||
"fixed_side_negative": "98 of 98 pairs (absent_fixed_side 0)"
|
||||
},
|
||||
"features": {
|
||||
"command": "benchmarks/learn/build_features.py --labels <the 392 new rows> --units <units.json of `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes-2/catalog.toml` (manifests written, no Run started)> --kinds pairs",
|
||||
"records": 392,
|
||||
"instrument_states_plane_all_families": {
|
||||
"agree:missing": 74,
|
||||
"agree:none": 122,
|
||||
"delta:not_applicable": 196,
|
||||
"engine:missing": 128,
|
||||
"engine:none": 68,
|
||||
"entry_points:not_applicable": 196,
|
||||
"facts:not_applicable": 196,
|
||||
"language:ran": 196,
|
||||
"model_sinks:missing": 196,
|
||||
"quick:ran": 196,
|
||||
"roles:ran": 196,
|
||||
"source:ran": 196,
|
||||
"verify:missing": 74,
|
||||
"verify:none": 122
|
||||
},
|
||||
"note": "quick rules and deterministic roles ran on each side; engine, verify, agree and model sinks never ran for these pairs (missing/none: null features, never zeros)"
|
||||
},
|
||||
"memory_build": {
|
||||
"command": "ousast learn memory build --labels <labels> --units <merged units.json> --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)",
|
||||
"examples": [
|
||||
937,
|
||||
1133
|
||||
],
|
||||
"pairs_examples": [
|
||||
772,
|
||||
968
|
||||
],
|
||||
"pairs_counts_after": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 968,
|
||||
"labels": 1936,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 968,
|
||||
"recorded": 968,
|
||||
"unread": 0
|
||||
}
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"how": "the 2026-10-02 gap record's count: static example rows of the store, per family, repository groups with a positive and no negative",
|
||||
"before": 99,
|
||||
"after": 99,
|
||||
"before_note": "the store before this harvest held 937 examples (the 10-02 gap record counted 904 and 98; the 10-02 rebuild added one population-v2 positive without a negative)",
|
||||
"why_unchanged": "the 99 are data gaps of other corpora (94 vibe-py pairs whose fixed side lost the function, 3 dev-php, 2 population-v2); a new repository group arrives with both labels, so it adds paired groups and leaves those counts as they were",
|
||||
"paired_groups_total": [
|
||||
151,
|
||||
249
|
||||
],
|
||||
"positives_only_share_of_positive_groups_six_families": [
|
||||
0.4034,
|
||||
0.2857
|
||||
],
|
||||
"by_family_before_after": {
|
||||
"access_control": {
|
||||
"groups_pos_only": [
|
||||
12,
|
||||
12
|
||||
],
|
||||
"groups_pos": [
|
||||
32,
|
||||
49
|
||||
],
|
||||
"groups_both": [
|
||||
20,
|
||||
37
|
||||
],
|
||||
"examples_neg": [
|
||||
20,
|
||||
37
|
||||
]
|
||||
},
|
||||
"config_secrets": {
|
||||
"groups_pos_only": [
|
||||
3,
|
||||
3
|
||||
],
|
||||
"groups_pos": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"groups_both": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"examples_neg": [
|
||||
13,
|
||||
13
|
||||
]
|
||||
},
|
||||
"deserialization": {
|
||||
"groups_pos_only": [
|
||||
20,
|
||||
20
|
||||
],
|
||||
"groups_pos": [
|
||||
41,
|
||||
52
|
||||
],
|
||||
"groups_both": [
|
||||
21,
|
||||
32
|
||||
],
|
||||
"examples_neg": [
|
||||
21,
|
||||
32
|
||||
]
|
||||
},
|
||||
"injection": {
|
||||
"groups_pos_only": [
|
||||
11,
|
||||
11
|
||||
],
|
||||
"groups_pos": [
|
||||
44,
|
||||
64
|
||||
],
|
||||
"groups_both": [
|
||||
33,
|
||||
53
|
||||
],
|
||||
"examples_neg": [
|
||||
70,
|
||||
90
|
||||
]
|
||||
},
|
||||
"memory": {
|
||||
"groups_pos_only": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"groups_pos": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"groups_both": [
|
||||
5,
|
||||
5
|
||||
],
|
||||
"examples_neg": [
|
||||
173,
|
||||
173
|
||||
]
|
||||
},
|
||||
"output_encoding": {
|
||||
"groups_pos_only": [
|
||||
14,
|
||||
14
|
||||
],
|
||||
"groups_pos": [
|
||||
36,
|
||||
52
|
||||
],
|
||||
"groups_both": [
|
||||
22,
|
||||
38
|
||||
],
|
||||
"examples_neg": [
|
||||
61,
|
||||
77
|
||||
]
|
||||
},
|
||||
"path": {
|
||||
"groups_pos_only": [
|
||||
12,
|
||||
12
|
||||
],
|
||||
"groups_pos": [
|
||||
33,
|
||||
50
|
||||
],
|
||||
"groups_both": [
|
||||
21,
|
||||
38
|
||||
],
|
||||
"examples_neg": [
|
||||
21,
|
||||
38
|
||||
]
|
||||
},
|
||||
"unknown": {
|
||||
"groups_pos_only": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"groups_pos": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"groups_both": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"examples_neg": [
|
||||
2,
|
||||
2
|
||||
]
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"groups_pos_only": [
|
||||
27,
|
||||
27
|
||||
],
|
||||
"groups_pos": [
|
||||
52,
|
||||
69
|
||||
],
|
||||
"groups_both": [
|
||||
25,
|
||||
42
|
||||
],
|
||||
"examples_neg": [
|
||||
36,
|
||||
53
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"embeddings": {
|
||||
"command": "ousast learn memory embed --budget-usd 0.20 (dry run first: 66,005 estimated tokens, $0.0013)",
|
||||
"requested": 1004,
|
||||
"cached": 781,
|
||||
"embedded": 223,
|
||||
"calls": 7,
|
||||
"tokens": 71163,
|
||||
"usd": 0.001423,
|
||||
"note": "embedded = the new pairs' excerpts plus the excerpts the 10-02 rebuild left without a vector; OpenRouter, the embedding model .env names"
|
||||
},
|
||||
"spend": {
|
||||
"chat_model_calls": 0,
|
||||
"embedding_usd": 0.001423,
|
||||
"github_api": "metadata and raw contents only"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,496 @@
|
||||
{
|
||||
"pairs": [
|
||||
{
|
||||
"name": "prefect-token-validation-e216171",
|
||||
"ok": true,
|
||||
"bytes_read": 3697
|
||||
},
|
||||
{
|
||||
"name": "arcadedb-getdatabasegroupsconfiguration-04110c0",
|
||||
"ok": true,
|
||||
"bytes_read": 3896
|
||||
},
|
||||
{
|
||||
"name": "langflow-get-flow-by-id-or-endpoint-name-2c9f498",
|
||||
"ok": true,
|
||||
"bytes_read": 3948
|
||||
},
|
||||
{
|
||||
"name": "grav-plugin-api-update-26f529c",
|
||||
"ok": true,
|
||||
"bytes_read": 5598
|
||||
},
|
||||
{
|
||||
"name": "google-maps-mcp-main-00d8725",
|
||||
"ok": true,
|
||||
"bytes_read": 3371
|
||||
},
|
||||
{
|
||||
"name": "camofox-mcp-createmcphttpapp-599f56e",
|
||||
"ok": true,
|
||||
"bytes_read": 5587
|
||||
},
|
||||
{
|
||||
"name": "spring-graphql-isoverridefor-63a89ed",
|
||||
"ok": true,
|
||||
"bytes_read": 2045
|
||||
},
|
||||
{
|
||||
"name": "jenkins-dofilltimezonenameitems-0586de4",
|
||||
"ok": true,
|
||||
"bytes_read": 2093
|
||||
},
|
||||
{
|
||||
"name": "netlicensing-mcp-dispatch-fbbb1d5",
|
||||
"ok": true,
|
||||
"bytes_read": 2610
|
||||
},
|
||||
{
|
||||
"name": "starlette-admin-validate-where-af05b45",
|
||||
"ok": true,
|
||||
"bytes_read": 2061
|
||||
},
|
||||
{
|
||||
"name": "next-auth-initauth-d008b9b",
|
||||
"ok": true,
|
||||
"bytes_read": 8193
|
||||
},
|
||||
{
|
||||
"name": "deepseek-mcp-server-starthttptransport-dab07ed",
|
||||
"ok": true,
|
||||
"bytes_read": 2394
|
||||
},
|
||||
{
|
||||
"name": "network-ai-routerequest-a59c13a",
|
||||
"ok": true,
|
||||
"bytes_read": 5945
|
||||
},
|
||||
{
|
||||
"name": "device-builder-parse-args-9e294f7",
|
||||
"ok": true,
|
||||
"bytes_read": 15243
|
||||
},
|
||||
{
|
||||
"name": "typespec-start-30d6f65",
|
||||
"ok": true,
|
||||
"bytes_read": 2109
|
||||
},
|
||||
{
|
||||
"name": "tomcat-findsecurityconstraints-8bafd79",
|
||||
"ok": true,
|
||||
"bytes_read": 18199
|
||||
},
|
||||
{
|
||||
"name": "vm2-makebuiltinsfromlegacyoptions-b0f5066",
|
||||
"ok": true,
|
||||
"bytes_read": 3539
|
||||
},
|
||||
{
|
||||
"name": "inlong-filtersensitive-ebcd6d3",
|
||||
"ok": true,
|
||||
"bytes_read": 6474
|
||||
},
|
||||
{
|
||||
"name": "h2o-3-validatejdbcurl-0298ee3",
|
||||
"ok": true,
|
||||
"bytes_read": 4919
|
||||
},
|
||||
{
|
||||
"name": "skops-load-model-29d61ea",
|
||||
"ok": true,
|
||||
"bytes_read": 1847
|
||||
},
|
||||
{
|
||||
"name": "picklescan-list-globals-58983e1",
|
||||
"ok": true,
|
||||
"bytes_read": 6975
|
||||
},
|
||||
{
|
||||
"name": "sglang-deserialize-49afb3d",
|
||||
"ok": true,
|
||||
"bytes_read": 4334
|
||||
},
|
||||
{
|
||||
"name": "pdfminer-six-load-data-b808ee0",
|
||||
"ok": true,
|
||||
"bytes_read": 2399
|
||||
},
|
||||
{
|
||||
"name": "reverb-handle-9ec26f8",
|
||||
"ok": true,
|
||||
"bytes_read": 3342
|
||||
},
|
||||
{
|
||||
"name": "transformers-is-kernel-a7f8e7f",
|
||||
"ok": true,
|
||||
"bytes_read": 1921
|
||||
},
|
||||
{
|
||||
"name": "opentelemetry-java-instrumentation-read-9cf4fba",
|
||||
"ok": true,
|
||||
"bytes_read": 2075
|
||||
},
|
||||
{
|
||||
"name": "keras-from-config-55888d3",
|
||||
"ok": true,
|
||||
"bytes_read": 3154
|
||||
},
|
||||
{
|
||||
"name": "airflow-handle-event-submit-a1ab065",
|
||||
"ok": true,
|
||||
"bytes_read": 4828
|
||||
},
|
||||
{
|
||||
"name": "sequelize-partialescape-5deadd2",
|
||||
"ok": true,
|
||||
"bytes_read": 6826
|
||||
},
|
||||
{
|
||||
"name": "electerm-mv-aa77881",
|
||||
"ok": true,
|
||||
"bytes_read": 1258
|
||||
},
|
||||
{
|
||||
"name": "systeminformation-checklinuxdchpinterfaces-bbfddde",
|
||||
"ok": true,
|
||||
"bytes_read": 2208
|
||||
},
|
||||
{
|
||||
"name": "prompty-buildagent-c27402d",
|
||||
"ok": true,
|
||||
"bytes_read": 3198
|
||||
},
|
||||
{
|
||||
"name": "datamodel-code-generator-get-field-extras-17fc235",
|
||||
"ok": true,
|
||||
"bytes_read": 2471
|
||||
},
|
||||
{
|
||||
"name": "spring-data-mongodb-bindablevaluefor-24d5afc",
|
||||
"ok": true,
|
||||
"bytes_read": 9895
|
||||
},
|
||||
{
|
||||
"name": "spring-ai-emitlucenestring-2b20cfc",
|
||||
"ok": true,
|
||||
"bytes_read": 4037
|
||||
},
|
||||
{
|
||||
"name": "aws-cdk-write-baa9e1d",
|
||||
"ok": true,
|
||||
"bytes_read": 1461
|
||||
},
|
||||
{
|
||||
"name": "danger-js-localgetfileatsha-087a729",
|
||||
"ok": true,
|
||||
"bytes_read": 1724
|
||||
},
|
||||
{
|
||||
"name": "argos-javascript-gitfetch-8355f3a",
|
||||
"ok": true,
|
||||
"bytes_read": 1319
|
||||
},
|
||||
{
|
||||
"name": "cakephp-transformfunctionexpression-138f2f6",
|
||||
"ok": true,
|
||||
"bytes_read": 6754
|
||||
},
|
||||
{
|
||||
"name": "orval-getroutepath-8ef1bfd",
|
||||
"ok": true,
|
||||
"bytes_read": 2607
|
||||
},
|
||||
{
|
||||
"name": "typeorm-handler-41d1c62",
|
||||
"ok": true,
|
||||
"bytes_read": 11977
|
||||
},
|
||||
{
|
||||
"name": "velocity-js-isblockedpathkey-f8e47a6",
|
||||
"ok": true,
|
||||
"bytes_read": 1539
|
||||
},
|
||||
{
|
||||
"name": "jsonata-evaluategroupexpression-4b217d5",
|
||||
"ok": true,
|
||||
"bytes_read": 7399
|
||||
},
|
||||
{
|
||||
"name": "nuxt-getislandcontext-5b60017",
|
||||
"ok": true,
|
||||
"bytes_read": 6805
|
||||
},
|
||||
{
|
||||
"name": "gitpython-option-candidates-96a888f",
|
||||
"ok": true,
|
||||
"bytes_read": 3421
|
||||
},
|
||||
{
|
||||
"name": "mermaid-addnamespace-12d472c",
|
||||
"ok": true,
|
||||
"bytes_read": 9722
|
||||
},
|
||||
{
|
||||
"name": "php-codesniffer-getblamecontent-7a3a6bb",
|
||||
"ok": true,
|
||||
"bytes_read": 2040
|
||||
},
|
||||
{
|
||||
"name": "xmldom-serializetostring-1cde3e3",
|
||||
"ok": true,
|
||||
"bytes_read": 23297
|
||||
},
|
||||
{
|
||||
"name": "trix-fromjson-9c0a993",
|
||||
"ok": true,
|
||||
"bytes_read": 1295
|
||||
},
|
||||
{
|
||||
"name": "oauthlib-prepare-token-revocation-request-c888359",
|
||||
"ok": true,
|
||||
"bytes_read": 4800
|
||||
},
|
||||
{
|
||||
"name": "suneditor-submitsrc-9d43a5e",
|
||||
"ok": true,
|
||||
"bytes_read": 2319
|
||||
},
|
||||
{
|
||||
"name": "mathlive-atomtomathml-5fe1c46",
|
||||
"ok": true,
|
||||
"bytes_read": 32529
|
||||
},
|
||||
{
|
||||
"name": "genql-comment-2dbc80f",
|
||||
"ok": true,
|
||||
"bytes_read": 1633
|
||||
},
|
||||
{
|
||||
"name": "angular-visitelementlike-417a407",
|
||||
"ok": true,
|
||||
"bytes_read": 8480
|
||||
},
|
||||
{
|
||||
"name": "cyberchef-objtotable-85db3be",
|
||||
"ok": true,
|
||||
"bytes_read": 2556
|
||||
},
|
||||
{
|
||||
"name": "mdc-isanchorlinkallowed-61d636c",
|
||||
"ok": true,
|
||||
"bytes_read": 1795
|
||||
},
|
||||
{
|
||||
"name": "node-xhtml-purifier-node-21d461a",
|
||||
"ok": true,
|
||||
"bytes_read": 7486
|
||||
},
|
||||
{
|
||||
"name": "orpc-init-daabded",
|
||||
"ok": true,
|
||||
"bytes_read": 6240
|
||||
},
|
||||
{
|
||||
"name": "mkdocs-material-mountsearchsuggest-52fb6be",
|
||||
"ok": true,
|
||||
"bytes_read": 4493
|
||||
},
|
||||
{
|
||||
"name": "jodit-isremovablenode-49a31f4",
|
||||
"ok": true,
|
||||
"bytes_read": 3921
|
||||
},
|
||||
{
|
||||
"name": "prowler-transform-6db407e",
|
||||
"ok": true,
|
||||
"bytes_read": 5481
|
||||
},
|
||||
{
|
||||
"name": "commonmark-parseattributes-dfcdf45",
|
||||
"ok": true,
|
||||
"bytes_read": 6437
|
||||
},
|
||||
{
|
||||
"name": "svgo-fn-3db3ef3",
|
||||
"ok": true,
|
||||
"bytes_read": 6895
|
||||
},
|
||||
{
|
||||
"name": "tiptap-mergeattributes-01d7af8",
|
||||
"ok": true,
|
||||
"bytes_read": 3690
|
||||
},
|
||||
{
|
||||
"name": "nogginlessdom-matchfilesnapshot-785e6ac",
|
||||
"ok": true,
|
||||
"bytes_read": 3023
|
||||
},
|
||||
{
|
||||
"name": "gradio-preprocess-97d541f",
|
||||
"ok": true,
|
||||
"bytes_read": 2581
|
||||
},
|
||||
{
|
||||
"name": "gemini-bridge-resolve-path-8f3b85a",
|
||||
"ok": true,
|
||||
"bytes_read": 2006
|
||||
},
|
||||
{
|
||||
"name": "libreoffice-convert-convertwithoptions-b78f17d",
|
||||
"ok": true,
|
||||
"bytes_read": 8912
|
||||
},
|
||||
{
|
||||
"name": "atomic-agents-stack-do-get-ec474f4",
|
||||
"ok": true,
|
||||
"bytes_read": 3493
|
||||
},
|
||||
{
|
||||
"name": "browse-mcp-downloadurl-5352a4a",
|
||||
"ok": true,
|
||||
"bytes_read": 5302
|
||||
},
|
||||
{
|
||||
"name": "notebooklm-mcp-makeslug-13828d9",
|
||||
"ok": true,
|
||||
"bytes_read": 2103
|
||||
},
|
||||
{
|
||||
"name": "langchain-nvidia-url-to-b64-string-b483c7c",
|
||||
"ok": true,
|
||||
"bytes_read": 2318
|
||||
},
|
||||
{
|
||||
"name": "redocly-cli-iteratepathitems-5041204",
|
||||
"ok": true,
|
||||
"bytes_read": 4412
|
||||
},
|
||||
{
|
||||
"name": "pip-filename-10dfb6b",
|
||||
"ok": true,
|
||||
"bytes_read": 1419
|
||||
},
|
||||
{
|
||||
"name": "lib-get-db-path-d665042",
|
||||
"ok": true,
|
||||
"bytes_read": 3879
|
||||
},
|
||||
{
|
||||
"name": "blender-mcp-download-polyhaven-asset-30a3308",
|
||||
"ok": true,
|
||||
"bytes_read": 35668
|
||||
},
|
||||
{
|
||||
"name": "datasets-set-feature-f989ef9",
|
||||
"ok": true,
|
||||
"bytes_read": 4882
|
||||
},
|
||||
{
|
||||
"name": "postcss-loadfile-7beca13",
|
||||
"ok": true,
|
||||
"bytes_read": 1712
|
||||
},
|
||||
{
|
||||
"name": "vitest-configureserver-51edf2b",
|
||||
"ok": true,
|
||||
"bytes_read": 3287
|
||||
},
|
||||
{
|
||||
"name": "next-js-getfilepath-968b9fc",
|
||||
"ok": true,
|
||||
"bytes_read": 2462
|
||||
},
|
||||
{
|
||||
"name": "composer-install-53b8bb4",
|
||||
"ok": true,
|
||||
"bytes_read": 4127
|
||||
},
|
||||
{
|
||||
"name": "vue-cli-mayproxy-72ba750",
|
||||
"ok": true,
|
||||
"bytes_read": 1203
|
||||
},
|
||||
{
|
||||
"name": "h3-redirectback-459a1c6",
|
||||
"ok": true,
|
||||
"bytes_read": 2750
|
||||
},
|
||||
{
|
||||
"name": "axios-setproxy-fb3befb",
|
||||
"ok": true,
|
||||
"bytes_read": 4403
|
||||
},
|
||||
{
|
||||
"name": "rembg-s-command-07ad0d4",
|
||||
"ok": true,
|
||||
"bytes_read": 21204
|
||||
},
|
||||
{
|
||||
"name": "http-server-getpreviousurl-2008fb6",
|
||||
"ok": true,
|
||||
"bytes_read": 4395
|
||||
},
|
||||
{
|
||||
"name": "authlib-get-authorization-grant-3be0846",
|
||||
"ok": true,
|
||||
"bytes_read": 2631
|
||||
},
|
||||
{
|
||||
"name": "java-client-overrideserverurl-2b9cd44",
|
||||
"ok": true,
|
||||
"bytes_read": 1382
|
||||
},
|
||||
{
|
||||
"name": "n8n-mcp-validatewebhookurl-9639f75",
|
||||
"ok": true,
|
||||
"bytes_read": 13458
|
||||
},
|
||||
{
|
||||
"name": "ufo-withoutbase-5cd9e67",
|
||||
"ok": true,
|
||||
"bytes_read": 2154
|
||||
},
|
||||
{
|
||||
"name": "symfony-dogenerate-4b63c3a",
|
||||
"ok": true,
|
||||
"bytes_read": 16981
|
||||
},
|
||||
{
|
||||
"name": "php-weasyprint-isoptionurl-9582dcf",
|
||||
"ok": true,
|
||||
"bytes_read": 1090
|
||||
},
|
||||
{
|
||||
"name": "compliance-trestle-check-metadata-endpoints-d107cd1",
|
||||
"ok": true,
|
||||
"bytes_read": 2412
|
||||
},
|
||||
{
|
||||
"name": "vllm-preprocess-audio-54503ec",
|
||||
"ok": true,
|
||||
"bytes_read": 4629
|
||||
},
|
||||
{
|
||||
"name": "phpspreadsheet-webservice-7ef7b25",
|
||||
"ok": true,
|
||||
"bytes_read": 4441
|
||||
},
|
||||
{
|
||||
"name": "frontmcp-dofetch-077201e",
|
||||
"ok": true,
|
||||
"bytes_read": 3737
|
||||
},
|
||||
{
|
||||
"name": "ip-address-islinklocal-d03e960",
|
||||
"ok": true,
|
||||
"bytes_read": 1489
|
||||
},
|
||||
{
|
||||
"name": "pyjwt-fetch-data-0a795b8",
|
||||
"ok": true,
|
||||
"bytes_read": 3385
|
||||
}
|
||||
],
|
||||
"verified": 98,
|
||||
"bytes_read": 526134
|
||||
}
|
||||
@@ -70,6 +70,18 @@ recorded = "2026-10-01"
|
||||
status = "development"
|
||||
exclude_tiers = ["title"]
|
||||
|
||||
# Batch 2: advisory fixes whose changed function survives on the fixed side, for the six compiled families (most
|
||||
# published after 2026-06-01). Same slice, cache layout and checks as batch 1; screening record
|
||||
# benchmarks/measurements/2026-10-03-decision-engine-negatives-harvest/record.json.
|
||||
[[source]]
|
||||
id = "advisory-fixes-2"
|
||||
kind = "pairs"
|
||||
file = "benchmarks/pairs/advisory-fixes-2/catalog.toml"
|
||||
evaluation = ["benchmarks/pairs/advisory-fixes-2/verification.json"]
|
||||
recorded = "2026-10-03"
|
||||
status = "development"
|
||||
exclude_tiers = ["title"]
|
||||
|
||||
[[source]]
|
||||
id = "adjudications"
|
||||
kind = "adjudications"
|
||||
|
||||
@@ -13,13 +13,16 @@ import tomllib
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learn.labels import DEFAULT_SOURCES, load_sources, repo_name
|
||||
from openultrasast.model.taxonomy import load_families
|
||||
from openultrasast.pairs import load_pair_catalog
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FOLDER = ROOT / "benchmarks" / "pairs" / "advisory-fixes"
|
||||
CATALOG = FOLDER / "catalog.toml"
|
||||
PAIRS = ROOT / "benchmarks" / "pairs"
|
||||
# batch 1 (2026-10-01) and batch 2 (2026-10-03, functions that survive the fix): one source each, same rules
|
||||
BATCHES = {"advisory-fixes": PAIRS / "advisory-fixes", "advisory-fixes-2": PAIRS / "advisory-fixes-2"}
|
||||
SHA = re.compile(r"[0-9a-f]{40}")
|
||||
PERMISSIVE_OR_COPYLEFT = re.compile(
|
||||
r"^(MIT|BSD-[23]-Clause|Apache-2\.0|ISC|Zlib|0BSD|Unlicense|MPL-[12]\.[01]"
|
||||
@@ -28,13 +31,18 @@ PERMISSIVE_OR_COPYLEFT = re.compile(
|
||||
GITHUB = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _raw() -> list[dict]:
|
||||
return list(tomllib.loads(CATALOG.read_text())["pair"])
|
||||
def _raw(folder: Path) -> list[dict]:
|
||||
return list(tomllib.loads((folder / "catalog.toml").read_text())["pair"])
|
||||
|
||||
|
||||
def test_every_pair_is_a_real_fix_with_its_function_licence_and_advisory() -> None:
|
||||
batches = pytest.mark.parametrize("source_id", sorted(BATCHES))
|
||||
|
||||
|
||||
@batches
|
||||
def test_every_pair_is_a_real_fix_with_its_function_licence_and_advisory(source_id: str) -> None:
|
||||
folder = BATCHES[source_id]
|
||||
families = {family.id for family in load_families().families}
|
||||
for item in _raw():
|
||||
for item in _raw(folder):
|
||||
name = item["name"]
|
||||
assert SHA.fullmatch(item["commit"]) and SHA.fullmatch(item["parent"]), name
|
||||
assert item["commit"] != item["parent"], f"{name}: the fix commit is its own parent"
|
||||
@@ -46,38 +54,45 @@ def test_every_pair_is_a_real_fix_with_its_function_licence_and_advisory() -> No
|
||||
assert item["vendored"] is False and item["slice"] == "advisory-fixes", name
|
||||
|
||||
|
||||
def test_the_catalog_loads_as_pointer_pairs_one_repository_each() -> None:
|
||||
cases = load_pair_catalog(CATALOG)
|
||||
assert len(cases) == len(_raw()) and all(not case.vendored for case in cases)
|
||||
@batches
|
||||
def test_the_catalog_loads_as_pointer_pairs_one_repository_each(source_id: str) -> None:
|
||||
folder = BATCHES[source_id]
|
||||
cases = load_pair_catalog(folder / "catalog.toml")
|
||||
assert len(cases) == len(_raw(folder)) and all(not case.vendored for case in cases)
|
||||
repos = Counter(repo_name(case.repo) for case in cases)
|
||||
assert [repo for repo, n in repos.items() if n > 1] == [], "one pair per repository"
|
||||
|
||||
|
||||
def test_the_label_builder_lists_the_catalog_as_a_source() -> None:
|
||||
@batches
|
||||
def test_the_label_builder_lists_the_catalog_as_a_source(source_id: str) -> None:
|
||||
sources = load_sources(DEFAULT_SOURCES)
|
||||
source = sources.get("advisory-fixes")
|
||||
assert source.kind == "pairs" and source.files == ("benchmarks/pairs/advisory-fixes/catalog.toml",)
|
||||
source = sources.get(source_id)
|
||||
assert source.kind == "pairs" and source.files == (f"benchmarks/pairs/{source_id}/catalog.toml",)
|
||||
assert all((ROOT / path).is_file() for path in source.evaluation)
|
||||
|
||||
|
||||
def test_every_pair_was_verified_on_both_sides() -> None:
|
||||
record = json.loads((FOLDER / "verification.json").read_text())
|
||||
@batches
|
||||
def test_every_pair_was_verified_on_both_sides(source_id: str) -> None:
|
||||
folder = BATCHES[source_id]
|
||||
record = json.loads((folder / "verification.json").read_text())
|
||||
rows = {row["name"]: row for row in record["pairs"]}
|
||||
for item in _raw():
|
||||
for item in _raw(folder):
|
||||
row = rows.get(item["name"])
|
||||
assert row is not None and row["ok"], f"{item['name']}: not verified ({row and row.get('reason')})"
|
||||
assert row["bytes_read"] > 0, f"{item['name']}: verified without reading anything"
|
||||
assert set(rows) == {item["name"] for item in _raw()}
|
||||
assert set(rows) == {item["name"] for item in _raw(folder)}
|
||||
|
||||
|
||||
def test_no_pair_repeats_a_repository_the_benchmarks_already_name() -> None:
|
||||
@batches
|
||||
def test_no_pair_repeats_a_repository_the_benchmarks_already_name(source_id: str) -> None:
|
||||
"""A repository group is counted once: no catalog repository appears anywhere else under benchmarks/ (files the
|
||||
label sources exclude, the reserved v3 population among them, are skipped by name and never opened)."""
|
||||
folder = BATCHES[source_id]
|
||||
excluded = load_sources(DEFAULT_SOURCES).excluded
|
||||
ours = {repo_name(item["repo"]) for item in _raw()}
|
||||
ours = {repo_name(item["repo"]) for item in _raw(folder)}
|
||||
elsewhere: set[str] = set()
|
||||
for path in (ROOT / "benchmarks").rglob("*"):
|
||||
if not path.is_file() or FOLDER in path.parents or path.name in excluded or path.suffix in {".pyc", ".bin", ".gz", ".jsonl"}:
|
||||
if not path.is_file() or folder in path.parents or path.name in excluded or path.suffix in {".pyc", ".bin", ".gz", ".jsonl"}:
|
||||
continue
|
||||
try:
|
||||
text = path.read_text(errors="ignore")
|
||||
@@ -86,3 +101,15 @@ def test_no_pair_repeats_a_repository_the_benchmarks_already_name() -> None:
|
||||
elsewhere |= {repo_name(match) for match in GITHUB.findall(text)}
|
||||
elsewhere |= {repo_name(m) for m in re.findall(r'^\s*(?:repo|fix_repo)\s*=\s*"([^"]+)"', text, re.MULTILINE)}
|
||||
assert sorted(ours & elsewhere) == []
|
||||
|
||||
|
||||
PERMISSIVE = {"MIT", "BSD-2-Clause", "BSD-3-Clause", "Apache-2.0", "ISC", "MPL-2.0", "Unlicense"}
|
||||
|
||||
|
||||
def test_batch_two_is_permissive_and_dated_against_the_cutoff() -> None:
|
||||
"""Batch 2 publishes permissive licences only and records each advisory's publication against the 2026-06-01
|
||||
cutoff, so memorisation can be measured on the post-cutoff pairs."""
|
||||
for item in _raw(BATCHES["advisory-fixes-2"]):
|
||||
assert item["license"] in PERMISSIVE, item["name"]
|
||||
assert item["batch"] == "2026-10-03", item["name"]
|
||||
assert item["post_cutoff"] is (item["published_at"][:10] >= "2026-06-01"), item["name"]
|
||||
|
||||
Reference in New Issue
Block a user