mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learn): candidate feature schema and builders (decision-engine task 1)
learn/schema.py is the closed, typed allow-list of candidate features: 66 features over 11 instruments (language one-hot, quick rules with 23 mechanism buckets and a prior count, engine, repo-facts, source span, entry-point distance, roles, delta, verify, agree, model sinks), two profiles (static 57; plane adds verify, agree and model sinks for 66), schema_version and a feature-set digest. No repository, path, file, function, rule or commit identity is a feature; site_match, in_fix_range, fixed-pin alerts, case.json fields and costs are refused. Missing is not zero: each record carries per-instrument state (ran | none | failed | not_applicable), and every feature of an instrument that did not run is null. learn/features.py builds records on the host (build_for_scan, whole repository or base..head delta); plane/tasks/features.py builds them per case in a Run from the delivered artifacts, dropping site_match and in_fix_range on read and ignoring fixed-pin alerts. The generator adds a model-free `features` entry per case before `remember`, which stores the records as `features` rows (re-validated). memory KINDS gains features, label, decision, experiment, arm_outcome and experiment_result. The committed validation-46 manifests are regenerated (entries appended; the first Run's entries are unchanged). The runner image must be rebuilt and re-pinned before a Run executes the new task. Proof on real artifacts (scratch store, not committed): the recorded validation-46 run gives 75 feature records over 15 cases (43 verify candidates plus vulnerable-pin alerts), each case's vulnerable pin exported and read (0.5k-9k files); remember stored 75 features rows beside 43 verdict, 84 alert, 61 unit_cost, 15 facts and 8 coverage rows. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
edcf145b00
commit
d1896cb8e7
@@ -5,7 +5,7 @@ Order follows the design's commit sequence. Every task is a commit gated on the
|
||||
adopted model every path is today's. Tasks 5, 7 and 10 spend model money or host hours: each starts only after
|
||||
the maintainer confirms its budget.
|
||||
|
||||
- [ ] 1. Schema and features
|
||||
- [x] 1. Schema and features
|
||||
- `learn/schema.py` (closed, typed allow-list; no identities; missing != zero), `learn/features.py` (host
|
||||
builder), new memory `KINDS`, `plane/tasks/features.py` and the `remember` input; tests enforcing the
|
||||
allow-list and the exclusion of label-carrying fields.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"modules": 135,
|
||||
"modules": 139,
|
||||
"counts": {
|
||||
"load_bearing": 128,
|
||||
"load_bearing": 132,
|
||||
"standalone_capability": 7
|
||||
},
|
||||
"orphaned": [],
|
||||
@@ -346,6 +346,33 @@
|
||||
"skills"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "learn",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
|
||||
"importers": [
|
||||
"learn.features",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "learn.features",
|
||||
"classification": "load_bearing",
|
||||
"reason": "imported by plane.tasks.features",
|
||||
"importers": [
|
||||
"plane.tasks.features"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "learn.schema",
|
||||
"classification": "load_bearing",
|
||||
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
|
||||
"importers": [
|
||||
"learn.features",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "map_gate",
|
||||
"classification": "load_bearing",
|
||||
@@ -361,6 +388,7 @@
|
||||
"gate",
|
||||
"hunter_tools",
|
||||
"improve.evolve",
|
||||
"learn.features",
|
||||
"map_gate",
|
||||
"model.scan",
|
||||
"pairs",
|
||||
@@ -386,6 +414,7 @@
|
||||
"cli",
|
||||
"cpg.backend",
|
||||
"cpg.session",
|
||||
"learn.features",
|
||||
"model.candidates",
|
||||
"model.config_value",
|
||||
"model.dominance",
|
||||
@@ -401,6 +430,7 @@
|
||||
"model.taint",
|
||||
"plane.budget",
|
||||
"plane.generate",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.repo_facts",
|
||||
"preprocess",
|
||||
"push.assistance",
|
||||
@@ -626,10 +656,12 @@
|
||||
"reason": "reachable from gate, map_gate, pair_gate or the CLI",
|
||||
"importers": [
|
||||
"cli",
|
||||
"learn.features",
|
||||
"model.candidates",
|
||||
"model.report",
|
||||
"model.scan",
|
||||
"model.specs"
|
||||
"model.specs",
|
||||
"plane.tasks.features"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -768,11 +800,13 @@
|
||||
"reason": "ai-service-plane Req 4.1: the task entrypoint package the runner imports by name; runner integration pending (task 3)",
|
||||
"importers": [
|
||||
"cli",
|
||||
"learn.features",
|
||||
"plane.engine_alerts",
|
||||
"plane.generate",
|
||||
"plane.memory",
|
||||
"plane.tasks.agree",
|
||||
"plane.tasks.alerts",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
@@ -787,9 +821,17 @@
|
||||
"classification": "load_bearing",
|
||||
"reason": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free",
|
||||
"importers": [
|
||||
"plane.engine_alerts"
|
||||
"learn.features",
|
||||
"plane.engine_alerts",
|
||||
"plane.tasks.features"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "plane.tasks.features",
|
||||
"classification": "load_bearing",
|
||||
"reason": "learned-decision-engine Req 1.1-1.3/7.2: one allow-listed feature record per candidate, model-free",
|
||||
"importers": []
|
||||
},
|
||||
{
|
||||
"module": "plane.tasks.loop",
|
||||
"classification": "load_bearing",
|
||||
@@ -811,8 +853,10 @@
|
||||
"classification": "load_bearing",
|
||||
"reason": "ai-service-plane Req 5.1/5.2: source-only functions and callers per file; runner integration pending",
|
||||
"importers": [
|
||||
"learn.features",
|
||||
"plane.generate",
|
||||
"plane.tasks.alerts",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.remember"
|
||||
]
|
||||
},
|
||||
@@ -872,6 +916,7 @@
|
||||
"hunter_tools",
|
||||
"improve.evolve",
|
||||
"index",
|
||||
"learn.features",
|
||||
"map_gate",
|
||||
"mapping",
|
||||
"model.candidates",
|
||||
@@ -879,6 +924,7 @@
|
||||
"model.scan",
|
||||
"pairs",
|
||||
"plane.tasks.alerts",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.repo_facts",
|
||||
"push.runner",
|
||||
"rank",
|
||||
@@ -1148,8 +1194,10 @@
|
||||
"improve.evolve",
|
||||
"improve.memory",
|
||||
"improve.validator",
|
||||
"learn.features",
|
||||
"pairs",
|
||||
"plane.tasks.alerts",
|
||||
"plane.tasks.features",
|
||||
"plane.tasks.loop",
|
||||
"stage_processors"
|
||||
]
|
||||
@@ -1229,6 +1277,7 @@
|
||||
"cli",
|
||||
"cpg.capability",
|
||||
"hunter_tools",
|
||||
"learn.features",
|
||||
"mapping",
|
||||
"model.candidates",
|
||||
"model.layout",
|
||||
@@ -1283,6 +1332,7 @@
|
||||
"importers": [
|
||||
"cli",
|
||||
"hunter_tools",
|
||||
"learn.features",
|
||||
"mapping",
|
||||
"model.candidates",
|
||||
"model.layout",
|
||||
|
||||
@@ -939,6 +939,24 @@ spec:
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: budibase-mongo-template-nosqli-features
|
||||
task: features-budibase-mongo-template-nosqli
|
||||
inputs:
|
||||
facts: budibase-mongo-template-nosqli-facts/facts.json
|
||||
facts_summary: budibase-mongo-template-nosqli-facts/summary.json
|
||||
pass_a: budibase-mongo-template-nosqli-va/units.jsonl
|
||||
summary_a: budibase-mongo-template-nosqli-va/summary.json
|
||||
pass_b: budibase-mongo-template-nosqli-vb/units.jsonl
|
||||
summary_b: budibase-mongo-template-nosqli-vb/summary.json
|
||||
pass_c: budibase-mongo-template-nosqli-vc/units.jsonl
|
||||
summary_c: budibase-mongo-template-nosqli-vc/summary.json
|
||||
agreed: budibase-mongo-template-nosqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: budibase-mongo-template-nosqli-remember
|
||||
task: remember-budibase-mongo-template-nosqli
|
||||
inputs:
|
||||
@@ -951,11 +969,30 @@ spec:
|
||||
pass_c: budibase-mongo-template-nosqli-vc/units.jsonl
|
||||
summary_c: budibase-mongo-template-nosqli-vc/summary.json
|
||||
agreed: budibase-mongo-template-nosqli-final/agreed.json
|
||||
features: budibase-mongo-template-nosqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: cct-studio-server-cmdi-features
|
||||
task: features-cct-studio-server-cmdi
|
||||
inputs:
|
||||
facts: cct-studio-server-cmdi-facts/facts.json
|
||||
facts_summary: cct-studio-server-cmdi-facts/summary.json
|
||||
pass_a: cct-studio-server-cmdi-va/units.jsonl
|
||||
summary_a: cct-studio-server-cmdi-va/summary.json
|
||||
pass_b: cct-studio-server-cmdi-vb/units.jsonl
|
||||
summary_b: cct-studio-server-cmdi-vb/summary.json
|
||||
pass_c: cct-studio-server-cmdi-vc/units.jsonl
|
||||
summary_c: cct-studio-server-cmdi-vc/summary.json
|
||||
agreed: cct-studio-server-cmdi-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: cct-studio-server-cmdi-remember
|
||||
task: remember-cct-studio-server-cmdi
|
||||
inputs:
|
||||
@@ -968,11 +1005,30 @@ spec:
|
||||
pass_c: cct-studio-server-cmdi-vc/units.jsonl
|
||||
summary_c: cct-studio-server-cmdi-vc/summary.json
|
||||
agreed: cct-studio-server-cmdi-final/agreed.json
|
||||
features: cct-studio-server-cmdi-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: contextforge-prompt-ssti-features
|
||||
task: features-contextforge-prompt-ssti
|
||||
inputs:
|
||||
facts: contextforge-prompt-ssti-facts/facts.json
|
||||
facts_summary: contextforge-prompt-ssti-facts/summary.json
|
||||
pass_a: contextforge-prompt-ssti-va/units.jsonl
|
||||
summary_a: contextforge-prompt-ssti-va/summary.json
|
||||
pass_b: contextforge-prompt-ssti-vb/units.jsonl
|
||||
summary_b: contextforge-prompt-ssti-vb/summary.json
|
||||
pass_c: contextforge-prompt-ssti-vc/units.jsonl
|
||||
summary_c: contextforge-prompt-ssti-vc/summary.json
|
||||
agreed: contextforge-prompt-ssti-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: contextforge-prompt-ssti-remember
|
||||
task: remember-contextforge-prompt-ssti
|
||||
inputs:
|
||||
@@ -985,11 +1041,30 @@ spec:
|
||||
pass_c: contextforge-prompt-ssti-vc/units.jsonl
|
||||
summary_c: contextforge-prompt-ssti-vc/summary.json
|
||||
agreed: contextforge-prompt-ssti-final/agreed.json
|
||||
features: contextforge-prompt-ssti-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: froxlor-admin-ip-second-order-sqli-features
|
||||
task: features-froxlor-admin-ip-second-order-sqli
|
||||
inputs:
|
||||
facts: froxlor-admin-ip-second-order-sqli-facts/facts.json
|
||||
facts_summary: froxlor-admin-ip-second-order-sqli-facts/summary.json
|
||||
pass_a: froxlor-admin-ip-second-order-sqli-va/units.jsonl
|
||||
summary_a: froxlor-admin-ip-second-order-sqli-va/summary.json
|
||||
pass_b: froxlor-admin-ip-second-order-sqli-vb/units.jsonl
|
||||
summary_b: froxlor-admin-ip-second-order-sqli-vb/summary.json
|
||||
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
|
||||
summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json
|
||||
agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: froxlor-admin-ip-second-order-sqli-remember
|
||||
task: remember-froxlor-admin-ip-second-order-sqli
|
||||
inputs:
|
||||
@@ -1002,11 +1077,30 @@ spec:
|
||||
pass_c: froxlor-admin-ip-second-order-sqli-vc/units.jsonl
|
||||
summary_c: froxlor-admin-ip-second-order-sqli-vc/summary.json
|
||||
agreed: froxlor-admin-ip-second-order-sqli-final/agreed.json
|
||||
features: froxlor-admin-ip-second-order-sqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: fuxa-tdengine-escape-sqli-features
|
||||
task: features-fuxa-tdengine-escape-sqli
|
||||
inputs:
|
||||
facts: fuxa-tdengine-escape-sqli-facts/facts.json
|
||||
facts_summary: fuxa-tdengine-escape-sqli-facts/summary.json
|
||||
pass_a: fuxa-tdengine-escape-sqli-va/units.jsonl
|
||||
summary_a: fuxa-tdengine-escape-sqli-va/summary.json
|
||||
pass_b: fuxa-tdengine-escape-sqli-vb/units.jsonl
|
||||
summary_b: fuxa-tdengine-escape-sqli-vb/summary.json
|
||||
pass_c: fuxa-tdengine-escape-sqli-vc/units.jsonl
|
||||
summary_c: fuxa-tdengine-escape-sqli-vc/summary.json
|
||||
agreed: fuxa-tdengine-escape-sqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: fuxa-tdengine-escape-sqli-remember
|
||||
task: remember-fuxa-tdengine-escape-sqli
|
||||
inputs:
|
||||
@@ -1019,11 +1113,30 @@ spec:
|
||||
pass_c: fuxa-tdengine-escape-sqli-vc/units.jsonl
|
||||
summary_c: fuxa-tdengine-escape-sqli-vc/summary.json
|
||||
agreed: fuxa-tdengine-escape-sqli-final/agreed.json
|
||||
features: fuxa-tdengine-escape-sqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: lmdeploy-media-url-ssrf-features
|
||||
task: features-lmdeploy-media-url-ssrf
|
||||
inputs:
|
||||
facts: lmdeploy-media-url-ssrf-facts/facts.json
|
||||
facts_summary: lmdeploy-media-url-ssrf-facts/summary.json
|
||||
pass_a: lmdeploy-media-url-ssrf-va/units.jsonl
|
||||
summary_a: lmdeploy-media-url-ssrf-va/summary.json
|
||||
pass_b: lmdeploy-media-url-ssrf-vb/units.jsonl
|
||||
summary_b: lmdeploy-media-url-ssrf-vb/summary.json
|
||||
pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl
|
||||
summary_c: lmdeploy-media-url-ssrf-vc/summary.json
|
||||
agreed: lmdeploy-media-url-ssrf-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: lmdeploy-media-url-ssrf-remember
|
||||
task: remember-lmdeploy-media-url-ssrf
|
||||
inputs:
|
||||
@@ -1036,11 +1149,30 @@ spec:
|
||||
pass_c: lmdeploy-media-url-ssrf-vc/units.jsonl
|
||||
summary_c: lmdeploy-media-url-ssrf-vc/summary.json
|
||||
agreed: lmdeploy-media-url-ssrf-final/agreed.json
|
||||
features: lmdeploy-media-url-ssrf-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: medplum-external-auth-open-redirect-features
|
||||
task: features-medplum-external-auth-open-redirect
|
||||
inputs:
|
||||
facts: medplum-external-auth-open-redirect-facts/facts.json
|
||||
facts_summary: medplum-external-auth-open-redirect-facts/summary.json
|
||||
pass_a: medplum-external-auth-open-redirect-va/units.jsonl
|
||||
summary_a: medplum-external-auth-open-redirect-va/summary.json
|
||||
pass_b: medplum-external-auth-open-redirect-vb/units.jsonl
|
||||
summary_b: medplum-external-auth-open-redirect-vb/summary.json
|
||||
pass_c: medplum-external-auth-open-redirect-vc/units.jsonl
|
||||
summary_c: medplum-external-auth-open-redirect-vc/summary.json
|
||||
agreed: medplum-external-auth-open-redirect-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: medplum-external-auth-open-redirect-remember
|
||||
task: remember-medplum-external-auth-open-redirect
|
||||
inputs:
|
||||
@@ -1053,11 +1185,30 @@ spec:
|
||||
pass_c: medplum-external-auth-open-redirect-vc/units.jsonl
|
||||
summary_c: medplum-external-auth-open-redirect-vc/summary.json
|
||||
agreed: medplum-external-auth-open-redirect-final/agreed.json
|
||||
features: medplum-external-auth-open-redirect-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: pgadmin-maintenance-sqli-features
|
||||
task: features-pgadmin-maintenance-sqli
|
||||
inputs:
|
||||
facts: pgadmin-maintenance-sqli-facts/facts.json
|
||||
facts_summary: pgadmin-maintenance-sqli-facts/summary.json
|
||||
pass_a: pgadmin-maintenance-sqli-va/units.jsonl
|
||||
summary_a: pgadmin-maintenance-sqli-va/summary.json
|
||||
pass_b: pgadmin-maintenance-sqli-vb/units.jsonl
|
||||
summary_b: pgadmin-maintenance-sqli-vb/summary.json
|
||||
pass_c: pgadmin-maintenance-sqli-vc/units.jsonl
|
||||
summary_c: pgadmin-maintenance-sqli-vc/summary.json
|
||||
agreed: pgadmin-maintenance-sqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: pgadmin-maintenance-sqli-remember
|
||||
task: remember-pgadmin-maintenance-sqli
|
||||
inputs:
|
||||
@@ -1070,11 +1221,30 @@ spec:
|
||||
pass_c: pgadmin-maintenance-sqli-vc/units.jsonl
|
||||
summary_c: pgadmin-maintenance-sqli-vc/summary.json
|
||||
agreed: pgadmin-maintenance-sqli-final/agreed.json
|
||||
features: pgadmin-maintenance-sqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: qwed-sympy-eval-features
|
||||
task: features-qwed-sympy-eval
|
||||
inputs:
|
||||
facts: qwed-sympy-eval-facts/facts.json
|
||||
facts_summary: qwed-sympy-eval-facts/summary.json
|
||||
pass_a: qwed-sympy-eval-va/units.jsonl
|
||||
summary_a: qwed-sympy-eval-va/summary.json
|
||||
pass_b: qwed-sympy-eval-vb/units.jsonl
|
||||
summary_b: qwed-sympy-eval-vb/summary.json
|
||||
pass_c: qwed-sympy-eval-vc/units.jsonl
|
||||
summary_c: qwed-sympy-eval-vc/summary.json
|
||||
agreed: qwed-sympy-eval-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: qwed-sympy-eval-remember
|
||||
task: remember-qwed-sympy-eval
|
||||
inputs:
|
||||
@@ -1087,11 +1257,30 @@ spec:
|
||||
pass_c: qwed-sympy-eval-vc/units.jsonl
|
||||
summary_c: qwed-sympy-eval-vc/summary.json
|
||||
agreed: qwed-sympy-eval-final/agreed.json
|
||||
features: qwed-sympy-eval-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: reactpress-findall-column-sqli-features
|
||||
task: features-reactpress-findall-column-sqli
|
||||
inputs:
|
||||
facts: reactpress-findall-column-sqli-facts/facts.json
|
||||
facts_summary: reactpress-findall-column-sqli-facts/summary.json
|
||||
pass_a: reactpress-findall-column-sqli-va/units.jsonl
|
||||
summary_a: reactpress-findall-column-sqli-va/summary.json
|
||||
pass_b: reactpress-findall-column-sqli-vb/units.jsonl
|
||||
summary_b: reactpress-findall-column-sqli-vb/summary.json
|
||||
pass_c: reactpress-findall-column-sqli-vc/units.jsonl
|
||||
summary_c: reactpress-findall-column-sqli-vc/summary.json
|
||||
agreed: reactpress-findall-column-sqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: reactpress-findall-column-sqli-remember
|
||||
task: remember-reactpress-findall-column-sqli
|
||||
inputs:
|
||||
@@ -1104,11 +1293,30 @@ spec:
|
||||
pass_c: reactpress-findall-column-sqli-vc/units.jsonl
|
||||
summary_c: reactpress-findall-column-sqli-vc/summary.json
|
||||
agreed: reactpress-findall-column-sqli-final/agreed.json
|
||||
features: reactpress-findall-column-sqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: unleash-addon-webhook-ssrf-features
|
||||
task: features-unleash-addon-webhook-ssrf
|
||||
inputs:
|
||||
facts: unleash-addon-webhook-ssrf-facts/facts.json
|
||||
facts_summary: unleash-addon-webhook-ssrf-facts/summary.json
|
||||
pass_a: unleash-addon-webhook-ssrf-va/units.jsonl
|
||||
summary_a: unleash-addon-webhook-ssrf-va/summary.json
|
||||
pass_b: unleash-addon-webhook-ssrf-vb/units.jsonl
|
||||
summary_b: unleash-addon-webhook-ssrf-vb/summary.json
|
||||
pass_c: unleash-addon-webhook-ssrf-vc/units.jsonl
|
||||
summary_c: unleash-addon-webhook-ssrf-vc/summary.json
|
||||
agreed: unleash-addon-webhook-ssrf-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: unleash-addon-webhook-ssrf-remember
|
||||
task: remember-unleash-addon-webhook-ssrf
|
||||
inputs:
|
||||
@@ -1121,11 +1329,30 @@ spec:
|
||||
pass_c: unleash-addon-webhook-ssrf-vc/units.jsonl
|
||||
summary_c: unleash-addon-webhook-ssrf-vc/summary.json
|
||||
agreed: unleash-addon-webhook-ssrf-final/agreed.json
|
||||
features: unleash-addon-webhook-ssrf-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: winml-cli-serve-cors-features
|
||||
task: features-winml-cli-serve-cors
|
||||
inputs:
|
||||
facts: winml-cli-serve-cors-facts/facts.json
|
||||
facts_summary: winml-cli-serve-cors-facts/summary.json
|
||||
pass_a: winml-cli-serve-cors-va/units.jsonl
|
||||
summary_a: winml-cli-serve-cors-va/summary.json
|
||||
pass_b: winml-cli-serve-cors-vb/units.jsonl
|
||||
summary_b: winml-cli-serve-cors-vb/summary.json
|
||||
pass_c: winml-cli-serve-cors-vc/units.jsonl
|
||||
summary_c: winml-cli-serve-cors-vc/summary.json
|
||||
agreed: winml-cli-serve-cors-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: winml-cli-serve-cors-remember
|
||||
task: remember-winml-cli-serve-cors
|
||||
inputs:
|
||||
@@ -1138,11 +1365,30 @@ spec:
|
||||
pass_c: winml-cli-serve-cors-vc/units.jsonl
|
||||
summary_c: winml-cli-serve-cors-vc/summary.json
|
||||
agreed: winml-cli-serve-cors-final/agreed.json
|
||||
features: winml-cli-serve-cors-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-directorist-avatar-ssrf-features
|
||||
task: features-wp-directorist-avatar-ssrf
|
||||
inputs:
|
||||
facts: wp-directorist-avatar-ssrf-facts/facts.json
|
||||
facts_summary: wp-directorist-avatar-ssrf-facts/summary.json
|
||||
pass_a: wp-directorist-avatar-ssrf-va/units.jsonl
|
||||
summary_a: wp-directorist-avatar-ssrf-va/summary.json
|
||||
pass_b: wp-directorist-avatar-ssrf-vb/units.jsonl
|
||||
summary_b: wp-directorist-avatar-ssrf-vb/summary.json
|
||||
pass_c: wp-directorist-avatar-ssrf-vc/units.jsonl
|
||||
summary_c: wp-directorist-avatar-ssrf-vc/summary.json
|
||||
agreed: wp-directorist-avatar-ssrf-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-directorist-avatar-ssrf-remember
|
||||
task: remember-wp-directorist-avatar-ssrf
|
||||
inputs:
|
||||
@@ -1155,11 +1401,30 @@ spec:
|
||||
pass_c: wp-directorist-avatar-ssrf-vc/units.jsonl
|
||||
summary_c: wp-directorist-avatar-ssrf-vc/summary.json
|
||||
agreed: wp-directorist-avatar-ssrf-final/agreed.json
|
||||
features: wp-directorist-avatar-ssrf-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-gopay-log-filter-sqli-features
|
||||
task: features-wp-gopay-log-filter-sqli
|
||||
inputs:
|
||||
facts: wp-gopay-log-filter-sqli-facts/facts.json
|
||||
facts_summary: wp-gopay-log-filter-sqli-facts/summary.json
|
||||
pass_a: wp-gopay-log-filter-sqli-va/units.jsonl
|
||||
summary_a: wp-gopay-log-filter-sqli-va/summary.json
|
||||
pass_b: wp-gopay-log-filter-sqli-vb/units.jsonl
|
||||
summary_b: wp-gopay-log-filter-sqli-vb/summary.json
|
||||
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
|
||||
summary_c: wp-gopay-log-filter-sqli-vc/summary.json
|
||||
agreed: wp-gopay-log-filter-sqli-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-gopay-log-filter-sqli-remember
|
||||
task: remember-wp-gopay-log-filter-sqli
|
||||
inputs:
|
||||
@@ -1172,11 +1437,30 @@ spec:
|
||||
pass_c: wp-gopay-log-filter-sqli-vc/units.jsonl
|
||||
summary_c: wp-gopay-log-filter-sqli-vc/summary.json
|
||||
agreed: wp-gopay-log-filter-sqli-final/agreed.json
|
||||
features: wp-gopay-log-filter-sqli-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-groundhogg-confirm-redirect-features
|
||||
task: features-wp-groundhogg-confirm-redirect
|
||||
inputs:
|
||||
facts: wp-groundhogg-confirm-redirect-facts/facts.json
|
||||
facts_summary: wp-groundhogg-confirm-redirect-facts/summary.json
|
||||
pass_a: wp-groundhogg-confirm-redirect-va/units.jsonl
|
||||
summary_a: wp-groundhogg-confirm-redirect-va/summary.json
|
||||
pass_b: wp-groundhogg-confirm-redirect-vb/units.jsonl
|
||||
summary_b: wp-groundhogg-confirm-redirect-vb/summary.json
|
||||
pass_c: wp-groundhogg-confirm-redirect-vc/units.jsonl
|
||||
summary_c: wp-groundhogg-confirm-redirect-vc/summary.json
|
||||
agreed: wp-groundhogg-confirm-redirect-final/agreed.json
|
||||
outputs:
|
||||
- features.jsonl
|
||||
- summary.json
|
||||
budget:
|
||||
usd: 0
|
||||
calls: 0
|
||||
- name: wp-groundhogg-confirm-redirect-remember
|
||||
task: remember-wp-groundhogg-confirm-redirect
|
||||
inputs:
|
||||
@@ -1189,6 +1473,7 @@ spec:
|
||||
pass_c: wp-groundhogg-confirm-redirect-vc/units.jsonl
|
||||
summary_c: wp-groundhogg-confirm-redirect-vc/summary.json
|
||||
agreed: wp-groundhogg-confirm-redirect-final/agreed.json
|
||||
features: wp-groundhogg-confirm-redirect-features/features.jsonl
|
||||
outputs:
|
||||
- memory.jsonl
|
||||
budget:
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# features (learned-decision-engine design section 1): one allow-listed feature record per candidate. No model.
|
||||
# The template of the per-case Tasks (bound to the case's vulnerable Workspace for the functions' spans and the
|
||||
# entry-point distance); `remember` stores the records as `features` rows. The image is digest-pinned.
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features
|
||||
atespace: default
|
||||
spec:
|
||||
image: "localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699"
|
||||
command: ["features"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: "250m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "1Gi"
|
||||
@@ -2236,6 +2236,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-budibase-mongo-template-nosqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: budibase-mongo-template-nosqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-budibase-mongo-template-nosqli
|
||||
atespace: default
|
||||
@@ -2261,6 +2284,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-cct-studio-server-cmdi
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: cct-studio-server-cmdi-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-cct-studio-server-cmdi
|
||||
atespace: default
|
||||
@@ -2286,6 +2332,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-contextforge-prompt-ssti
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: contextforge-prompt-ssti-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-contextforge-prompt-ssti
|
||||
atespace: default
|
||||
@@ -2311,6 +2380,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-froxlor-admin-ip-second-order-sqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: froxlor-admin-ip-second-order-sqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-froxlor-admin-ip-second-order-sqli
|
||||
atespace: default
|
||||
@@ -2336,6 +2428,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-fuxa-tdengine-escape-sqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: fuxa-tdengine-escape-sqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-fuxa-tdengine-escape-sqli
|
||||
atespace: default
|
||||
@@ -2361,6 +2476,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-lmdeploy-media-url-ssrf
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: lmdeploy-media-url-ssrf-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-lmdeploy-media-url-ssrf
|
||||
atespace: default
|
||||
@@ -2386,6 +2524,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-medplum-external-auth-open-redirect
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: medplum-external-auth-open-redirect-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-medplum-external-auth-open-redirect
|
||||
atespace: default
|
||||
@@ -2411,6 +2572,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-pgadmin-maintenance-sqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: pgadmin-maintenance-sqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-pgadmin-maintenance-sqli
|
||||
atespace: default
|
||||
@@ -2436,6 +2620,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-qwed-sympy-eval
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: qwed-sympy-eval-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-qwed-sympy-eval
|
||||
atespace: default
|
||||
@@ -2461,6 +2668,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-reactpress-findall-column-sqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: reactpress-findall-column-sqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-reactpress-findall-column-sqli
|
||||
atespace: default
|
||||
@@ -2486,6 +2716,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-unleash-addon-webhook-ssrf
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: unleash-addon-webhook-ssrf-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-unleash-addon-webhook-ssrf
|
||||
atespace: default
|
||||
@@ -2511,6 +2764,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-winml-cli-serve-cors
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: winml-cli-serve-cors-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-winml-cli-serve-cors
|
||||
atespace: default
|
||||
@@ -2536,6 +2812,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-wp-directorist-avatar-ssrf
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: wp-directorist-avatar-ssrf-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-wp-directorist-avatar-ssrf
|
||||
atespace: default
|
||||
@@ -2561,6 +2860,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-wp-gopay-log-filter-sqli
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: wp-gopay-log-filter-sqli-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-wp-gopay-log-filter-sqli
|
||||
atespace: default
|
||||
@@ -2586,6 +2908,29 @@ spec:
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: features-wp-groundhogg-confirm-redirect
|
||||
atespace: default
|
||||
spec:
|
||||
image: localhost:5001/ousast-runner@sha256:e9964977659c4dcea7d8b25390ca0e16368ac901dd076aa6138df8303a0bb699
|
||||
command:
|
||||
- features
|
||||
env:
|
||||
- name: OUSAST_WORKSPACE_DIR
|
||||
value: /workspace/case/repo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
workspaces:
|
||||
- name: wp-groundhogg-confirm-redirect-vulnerable
|
||||
path: /workspace/case
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: remember-wp-groundhogg-confirm-redirect
|
||||
atespace: default
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""The learned decision engine (learned-decision-engine spec): instruments emit signals, a model decides.
|
||||
|
||||
Task 1 ships the candidate feature schema (:mod:`.schema`) and the host feature builder (:mod:`.features`); labels,
|
||||
training, calibration and the decision stage follow in later tasks.
|
||||
"""
|
||||
@@ -0,0 +1,540 @@
|
||||
"""Feature records per candidate, from the signals every instrument emitted (learned-decision-engine design section 1).
|
||||
|
||||
No model. The same per-instrument builders serve two callers:
|
||||
|
||||
- the plane task ``features`` (:mod:`..plane.tasks.features`), which reads a case's delivered artifacts;
|
||||
- the host: :func:`build_for_scan` over a checkout's quick findings and an engine result, for ``scan`` and
|
||||
``pre-push`` (a whole-repository scan, or a base..head delta with :class:`Delta`).
|
||||
|
||||
A candidate is ``(path, function, family)`` of one repository and pin: a function where at least one instrument
|
||||
emitted a signal of that family. The key stays in the record's envelope (``candidate`` is ``path::function``); the
|
||||
features never see it (:mod:`.schema`). Each instrument contributes a :class:`Part` -- its state, its values and its
|
||||
version -- and :func:`record` assembles and validates the record: an instrument that did not run leaves ``null``,
|
||||
never a zero.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..model.taxonomy import load_families
|
||||
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
|
||||
from ..preprocess import build_file_target, detect_language
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
||||
from .schema import (
|
||||
BY_NAME,
|
||||
LANGUAGES,
|
||||
MECHANISMS,
|
||||
RULE_TAG_MECHANISM,
|
||||
RUNGS,
|
||||
SCHEMA_VERSION,
|
||||
FeatureRecordError,
|
||||
feature_set_digest,
|
||||
features_for,
|
||||
instruments_for,
|
||||
validate_record,
|
||||
)
|
||||
|
||||
ENGINE_RULE_PREFIX = "engine:"
|
||||
UNKNOWN_FAMILY = "unknown"
|
||||
# The operation a sink performs, by the CWE its semantic entry binds (a closed table; anything else is `other`).
|
||||
CWE_OPERATION = {
|
||||
"CWE-89": "sql", "CWE-943": "sql", "CWE-78": "command", "CWE-77": "command", "CWE-94": "code_eval", "CWE-95": "code_eval",
|
||||
"CWE-502": "deserialize", "CWE-22": "file_path", "CWE-98": "file_inclusion", "CWE-918": "outbound_request",
|
||||
"CWE-601": "redirect", "CWE-79": "html_output",
|
||||
} # fmt: skip
|
||||
# A semantic source entry's id mapped to its kind; an unlisted id is `other`.
|
||||
SOURCE_KIND = {
|
||||
"request": "request", "django-request": "request", "req": "request", "node-request": "request", "servlet": "request",
|
||||
"server": "server", "raw_body": "raw_body", "argv": "argv_stdin", "stdin": "argv_stdin", "input": "argv_stdin",
|
||||
} # fmt: skip
|
||||
_STEPS = re.compile(r"(\d+) steps")
|
||||
_WITNESS = re.compile(r"^(?P<source>.+?) -> (?P<sink>.+?)(?: in \S+)? \(line")
|
||||
_CALL = re.compile(r"(?<![A-Za-z0-9_$])([A-Za-z_]\w*)\s*\(")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Part:
|
||||
"""One instrument's contribution to a record: its state, its feature values (only when ``ran``), its version."""
|
||||
|
||||
state: str
|
||||
values: Mapping[str, Any] = field(default_factory=dict)
|
||||
version: str | None = None
|
||||
|
||||
|
||||
NONE = Part("none")
|
||||
NOT_APPLICABLE = Part("not_applicable")
|
||||
|
||||
|
||||
def language_of(path: str) -> str:
|
||||
"""The closed language of a path (its extension), ``other`` outside :data:`.schema.LANGUAGES`."""
|
||||
language = detect_language(Path(path))
|
||||
return language if language in LANGUAGES else "other"
|
||||
|
||||
|
||||
def _clip(name: str, value: Any) -> Any:
|
||||
spec = BY_NAME[name]
|
||||
if value is None or spec.type != "int" or isinstance(value, bool):
|
||||
return value
|
||||
return max(0, min(int(value), spec.cap or 0))
|
||||
|
||||
|
||||
def record(
|
||||
candidate: str,
|
||||
family: str,
|
||||
language: str,
|
||||
profile: str,
|
||||
parts: Mapping[str, Part],
|
||||
*,
|
||||
unit: str = "pin",
|
||||
base: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""The validated record of one candidate. ``language`` sets the one-hot; an instrument absent from ``parts`` is
|
||||
``none``; a value for a feature of another instrument, or outside the profile, is refused."""
|
||||
closed = language if language in LANGUAGES else "other"
|
||||
parts = {**parts, "language": Part("ran", {f"lang.{name}": name == closed for name in LANGUAGES})}
|
||||
wanted = instruments_for(profile)
|
||||
stray = sorted(set(parts) - set(wanted))
|
||||
if stray:
|
||||
raise FeatureRecordError(f"parts for instruments outside the {profile} profile: {stray}")
|
||||
x: dict[str, Any] = {}
|
||||
for spec in features_for(profile):
|
||||
part = parts.get(spec.instrument, NONE)
|
||||
x[spec.name] = _clip(spec.name, part.values.get(spec.name)) if part.state == "ran" else None
|
||||
for name in wanted:
|
||||
part = parts.get(name, NONE)
|
||||
foreign = sorted(k for k in part.values if k not in BY_NAME or BY_NAME[k].instrument != name)
|
||||
if foreign:
|
||||
raise FeatureRecordError(f"instrument {name} supplied features it does not own: {foreign}")
|
||||
instruments = {name: {"state": parts.get(name, NONE).state, "version": parts.get(name, NONE).version} for name in wanted}
|
||||
out: dict[str, Any] = {
|
||||
"candidate": candidate, "family": family, "language": closed, "profile": profile, "schema_version": SCHEMA_VERSION,
|
||||
"feature_set_digest": feature_set_digest(), "unit": unit, "x": x, "instruments": instruments,
|
||||
} # fmt: skip
|
||||
if base is not None:
|
||||
out["base"] = base
|
||||
return validate_record(out)
|
||||
|
||||
|
||||
# --- per-instrument parts --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ruleset_digest(rules: Iterable[PatternRule]) -> str:
|
||||
"""sha256 of the rules as data: the quick instrument's version."""
|
||||
canonical = json.dumps([asdict(rule) for rule in rules], sort_keys=True, separators=(",", ":"))
|
||||
return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _is_prior(item: object) -> bool:
|
||||
return bool(getattr(item, "framework", None) or getattr(item, "library", None))
|
||||
|
||||
|
||||
def quick_part(hits: Sequence[tuple[str, str]], rules: Mapping[str, PatternRule], version: str | None = None) -> Part:
|
||||
"""``hits`` are ``(rule_id, status)`` of the quick rules that fired in the candidate's function. Rule ids reach
|
||||
the record only as mechanism buckets and the prior count."""
|
||||
mechanisms = dict.fromkeys(MECHANISMS, 0)
|
||||
enabled = shadow = priors = 0
|
||||
precision: float | None = None
|
||||
for rule_id, status in hits:
|
||||
if status == "enabled":
|
||||
enabled += 1
|
||||
else:
|
||||
shadow += 1
|
||||
rule = rules.get(rule_id)
|
||||
if rule is None:
|
||||
mechanisms["other"] += 1
|
||||
continue
|
||||
for mechanism in sorted({RULE_TAG_MECHANISM.get(tag, "other") for tag in rule.tags} or {"other"}):
|
||||
mechanisms[mechanism] += 1
|
||||
precision = rule.precision_estimate if precision is None else max(precision, rule.precision_estimate)
|
||||
priors += 1 if _is_prior(rule) else 0
|
||||
values: dict[str, Any] = {"qr.enabled_hits": enabled, "qr.shadow_hits": shadow, "qr.prior_hits": priors}
|
||||
values.update({f"qr.mechanism.{m}": n for m, n in mechanisms.items()})
|
||||
values["qr.max_precision_estimate"] = None if precision is None else round(min(max(precision, 0.0), 1.0), 6)
|
||||
return Part("ran", values, version)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Vocabulary:
|
||||
"""The semantic entries of one language, to name a witness's sink operation and source kind and their origin."""
|
||||
|
||||
sinks: Mapping[str, tuple[str, str]] # call -> (operation, origin)
|
||||
sources: tuple[tuple[str, str, str], ...] # (pattern, kind, origin)
|
||||
|
||||
@classmethod
|
||||
def load(cls, language: str) -> Vocabulary:
|
||||
from ..semantic.facts import FactLoadError, load_facts
|
||||
|
||||
try:
|
||||
facts = load_facts().for_language(language)
|
||||
except FactLoadError:
|
||||
return cls({}, ())
|
||||
sinks: dict[str, tuple[str, str]] = {}
|
||||
for sink in facts.sinks:
|
||||
for call in sink.calls:
|
||||
sinks.setdefault(call, (CWE_OPERATION.get(sink.cwe, "other"), "prior" if _is_prior(sink) else "language"))
|
||||
sources = tuple(
|
||||
(pattern, SOURCE_KIND.get(src.id, "other"), "prior" if _is_prior(src) else "language")
|
||||
for src in facts.sources
|
||||
for pattern in src.patterns
|
||||
)
|
||||
return cls(sinks, sources)
|
||||
|
||||
def sink(self, text: str) -> tuple[str, str] | None:
|
||||
name = text.strip()
|
||||
for call in sorted(self.sinks, key=len, reverse=True):
|
||||
if name == call or name.endswith(("." + call, "->" + call, "::" + call)) or name.startswith(call + "("):
|
||||
return self.sinks[call]
|
||||
return None
|
||||
|
||||
def source(self, text: str) -> tuple[str, str] | None:
|
||||
for pattern, kind, origin in sorted(self.sources, key=lambda s: len(s[0]), reverse=True):
|
||||
if pattern in text:
|
||||
return kind, origin
|
||||
return None
|
||||
|
||||
|
||||
def engine_part(
|
||||
findings: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
completion: float | None,
|
||||
degraded: bool,
|
||||
vocabulary: Vocabulary | None = None,
|
||||
unstable: bool | None = None,
|
||||
version: str | None = None,
|
||||
) -> Part:
|
||||
"""The engine's findings in the candidate's function (``rung`` and ``witness`` when the producer kept them)."""
|
||||
rungs = [str(f["rung"]) for f in findings if str(f.get("rung")) in RUNGS]
|
||||
steps = [int(m.group(1)) for f in findings if (m := _STEPS.search(str(f.get("witness") or "")))]
|
||||
sink = source = None
|
||||
for finding in findings:
|
||||
match = _WITNESS.match(str(finding.get("witness") or ""))
|
||||
if match and vocabulary is not None:
|
||||
sink = sink or vocabulary.sink(match.group("sink"))
|
||||
source = source or vocabulary.source(match.group("source"))
|
||||
values = {
|
||||
"eng.findings": len(findings), "eng.rung_max": max(rungs, key=RUNGS.index) if rungs else None,
|
||||
"eng.witness_steps_min": min(steps) if steps else None, "eng.sink_kind": sink[0] if sink else None,
|
||||
"eng.sink_origin": sink[1] if sink else None, "eng.source_kind": source[0] if source else None,
|
||||
"eng.source_origin": source[1] if source else None, "eng.sanitizer_on_path": None,
|
||||
"eng.completion": None if completion is None else round(min(max(completion, 0.0), 1.0), 6), "eng.degraded": degraded,
|
||||
"eng.unstable": unstable,
|
||||
} # fmt: skip
|
||||
return Part("ran", values, version)
|
||||
|
||||
|
||||
def facts_part(callers: Sequence[Mapping[str, Any]], function: str, version: str | None = None) -> Part:
|
||||
"""``callers`` are the repo-facts call sites of the candidate's function in other files."""
|
||||
files = {str(site.get("path")) for site in callers}
|
||||
return Part("ran", {"facts.callers": len(callers), "facts.caller_files": len(files), "facts.is_global": function == GLOBAL}, version)
|
||||
|
||||
|
||||
def function_span(lines: Sequence[str], language: str, function: str) -> int | None:
|
||||
"""Lines of the first declaration of ``function`` up to the next declaration (Python: up to the next line indented
|
||||
no deeper), ``None`` when the file does not declare it. ``<global>`` counts the whole file."""
|
||||
if function == GLOBAL:
|
||||
return len(lines)
|
||||
pattern = DECLARATION.get(language)
|
||||
if pattern is None:
|
||||
return None
|
||||
start = next((i for i, line in enumerate(lines) if _declared_name(pattern, line) == function), None)
|
||||
if start is None:
|
||||
return None
|
||||
indent = len(lines[start]) - len(lines[start].lstrip(" \t"))
|
||||
for end in range(start + 1, len(lines)):
|
||||
line = lines[end]
|
||||
if language == "python":
|
||||
if line.strip() and not line.lstrip().startswith("#") and len(line) - len(line.lstrip(" \t")) <= indent:
|
||||
return end - start
|
||||
elif _declared_name(pattern, line):
|
||||
return end - start
|
||||
return len(lines) - start
|
||||
|
||||
|
||||
def source_part(lines: Sequence[str] | None, language: str, function: str) -> Part:
|
||||
if lines is None:
|
||||
return NONE
|
||||
span = function_span(lines, language, function)
|
||||
return Part("ran", {"facts.function_lines": span}) if span is not None else Part("failed")
|
||||
|
||||
|
||||
def verify_part(passes: Mapping[str, Sequence[Mapping[str, Any]]], candidate: str, line: int | None, version: str | None = None) -> Part:
|
||||
"""Pass verdicts of one candidate from the passes' ``units.jsonl``: asked in a pass when a unit's candidates list
|
||||
it, flagged when that unit's ``flagged`` names it. ``none`` when no pass asked it."""
|
||||
flags: dict[str, bool | None] = {}
|
||||
turns: list[float] = []
|
||||
offsets: list[int] = []
|
||||
for label in ("a", "b", "c"):
|
||||
asked = [u for u in passes.get(label) or () if any(f"{c[0]}::{c[1]}" == candidate for c in u.get("candidates") or ())]
|
||||
if not asked:
|
||||
flags[label] = None
|
||||
continue
|
||||
flagged = [f for u in asked for f in u.get("flagged") or () if f.get("candidate") == candidate]
|
||||
flags[label] = bool(flagged)
|
||||
turns.extend(float(u["turns"]) for u in asked if isinstance(u.get("turns"), (int, float)))
|
||||
for finding in flagged:
|
||||
reported = str(finding.get("reported_at") or "").rpartition(":")[2]
|
||||
if line is not None and reported.isdigit():
|
||||
offsets.append(abs(int(reported) - line))
|
||||
if flags["a"] is None and flags["b"] is None:
|
||||
return NONE
|
||||
values = {
|
||||
"verify.flag_a": flags["a"], "verify.flag_b": flags["b"], "verify.flag_c": flags["c"],
|
||||
"verify.votes": sum(1 for v in flags.values() if v), "verify.turns_mean": round(sum(turns) / len(turns), 3) if turns else None,
|
||||
"verify.site_offset": min(offsets) if offsets else None,
|
||||
} # fmt: skip
|
||||
return Part("ran", values, version)
|
||||
|
||||
|
||||
def agree_part(final: str | None) -> Part:
|
||||
return Part("ran", {"agree.final": final}) if final else NONE
|
||||
|
||||
|
||||
# --- the host path ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Delta:
|
||||
"""A base..head delta: the base checkout, the head-side changed line ranges per path (``git diff -U0``), and the
|
||||
base's quick findings and engine result, whose signals decide novelty."""
|
||||
|
||||
root: Path
|
||||
changed: Mapping[str, Sequence[tuple[int, int]]]
|
||||
findings: Sequence[Any] = ()
|
||||
engine_result: Mapping[str, Any] | None = None
|
||||
pin: str | None = None
|
||||
|
||||
|
||||
class SourceIndex:
|
||||
"""A checkout's product files read once: lines per path, the enclosing function of a line, and the name-level
|
||||
caller graph (the repo-facts patterns) for the entry-point distance."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
self._lines: dict[str, list[str] | None] = {}
|
||||
self._callers: dict[str, set[str]] | None = None
|
||||
self._files: list[str] | None = None
|
||||
|
||||
def files(self) -> list[str]:
|
||||
if self._files is None:
|
||||
self._files = [p.relative_to(self.root).as_posix() for p in product_files(self.root)]
|
||||
return self._files
|
||||
|
||||
def lines(self, path: str) -> list[str] | None:
|
||||
if path not in self._lines:
|
||||
file = self.root / path
|
||||
self._lines[path] = file.read_text(encoding="utf-8", errors="replace").splitlines() if file.is_file() else None
|
||||
return self._lines[path]
|
||||
|
||||
def function_at(self, path: str, line: int | None, fallback: str | None = None) -> str:
|
||||
lines = self.lines(path)
|
||||
language = detect_language(Path(path))
|
||||
if lines and language in DECLARATION and line and 0 < line <= len(lines):
|
||||
return enclosing(lines, line - 1, language)
|
||||
return fallback or GLOBAL
|
||||
|
||||
def callers(self) -> dict[str, set[str]]:
|
||||
"""Function name -> the names of the functions that call it anywhere in the product files."""
|
||||
if self._callers is None:
|
||||
declared: dict[str, set[str]] = {}
|
||||
files = self.files()
|
||||
for path in files:
|
||||
language = detect_language(Path(path))
|
||||
pattern = DECLARATION.get(language)
|
||||
if pattern is not None:
|
||||
declared.setdefault(language, set()).update(
|
||||
n for line in self.lines(path) or () if (n := _declared_name(pattern, line))
|
||||
)
|
||||
graph: dict[str, set[str]] = {}
|
||||
for path in files:
|
||||
language = detect_language(Path(path))
|
||||
names, lines = declared.get(language, set()), self.lines(path) or []
|
||||
for index, line in enumerate(lines):
|
||||
if line.lstrip().startswith(_COMMENT_PREFIXES):
|
||||
continue
|
||||
for match in _CALL.finditer(line):
|
||||
if match.group(1) in names:
|
||||
graph.setdefault(match.group(1), set()).add(enclosing(lines, index, language))
|
||||
self._callers = graph
|
||||
return self._callers
|
||||
|
||||
|
||||
def entry_distance(function: str, entries: set[str], callers: Mapping[str, set[str]], cap: int = 6) -> int:
|
||||
"""Caller-graph hops from an entry point to ``function``; ``cap`` means not reached within ``cap - 1`` hops."""
|
||||
frontier, seen = {function}, {function}
|
||||
for hops in range(cap):
|
||||
if frontier & entries:
|
||||
return hops
|
||||
frontier = {c for name in frontier for c in callers.get(name, ()) if c not in seen}
|
||||
seen |= frontier
|
||||
if not frontier:
|
||||
break
|
||||
return cap
|
||||
|
||||
|
||||
def entry_names(root: Path, paths: Iterable[str]) -> tuple[set[str], set[str]]:
|
||||
"""(entry function names, paths with a file-level entry) by :func:`..mapping.analyze_entry_points`."""
|
||||
from ..mapping import analyze_entry_points
|
||||
|
||||
targets = [build_file_target(root, root / p) for p in sorted(set(paths)) if (root / p).is_file()]
|
||||
records = analyze_entry_points(root, targets)
|
||||
return {r.function_name for r in records if r.function_name}, {r.path for r in records}
|
||||
|
||||
|
||||
def _family_of(rule: PatternRule | None) -> str:
|
||||
if rule is None:
|
||||
return UNKNOWN_FAMILY
|
||||
family = load_families().family_of_cwe(rule.cwe)
|
||||
return family.id if family is not None else UNKNOWN_FAMILY
|
||||
|
||||
|
||||
def _engine_state(result: Mapping[str, Any] | None) -> str:
|
||||
if result is None:
|
||||
return "none"
|
||||
return "ran" if int(result.get("questions") or 0) > 0 else "failed"
|
||||
|
||||
|
||||
def _signals(
|
||||
index: SourceIndex, findings: Sequence[Any], engine_result: Mapping[str, Any] | None, rules: Mapping[str, PatternRule]
|
||||
) -> tuple[dict[tuple[str, str, str], list[tuple[str, str]]], dict[tuple[str, str, str], list[Mapping[str, Any]]]]:
|
||||
quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {}
|
||||
for finding in findings:
|
||||
rule_id = str(finding.finding_id).split(":", 1)[0]
|
||||
function = index.function_at(finding.path, finding.line, finding.function_name)
|
||||
quick.setdefault((finding.path, function, _family_of(rules.get(rule_id))), []).append((rule_id, str(finding.status)))
|
||||
engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {}
|
||||
for finding in (engine_result or {}).get("findings") or []:
|
||||
path, _, rest = str(finding.get("site") or "").partition(":")
|
||||
line_text, _, function_name = rest.partition(":")
|
||||
line = int(line_text) if line_text.isdigit() else None
|
||||
function = index.function_at(path, line, function_name or None)
|
||||
engine.setdefault((path, function, str(finding.get("family") or UNKNOWN_FAMILY)), []).append(finding)
|
||||
return quick, engine
|
||||
|
||||
|
||||
def _changed_in(index: SourceIndex, changed: Mapping[str, Sequence[tuple[int, int]]]) -> dict[tuple[str, str], int]:
|
||||
"""(path, function) -> changed lines inside it, by the head's enclosing declarations."""
|
||||
counts: dict[tuple[str, str], int] = {}
|
||||
for path, ranges in changed.items():
|
||||
for lo, hi in ranges:
|
||||
for line in range(int(lo), int(hi) + 1):
|
||||
key = (path, index.function_at(path, line))
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _body_calls(lines: Sequence[str] | None, language: str, function: str) -> set[str]:
|
||||
if not lines:
|
||||
return set()
|
||||
return {m.group(1) for i, line in enumerate(lines) if enclosing(lines, i, language) == function for m in _CALL.finditer(line)}
|
||||
|
||||
|
||||
def build_for_scan(
|
||||
root: Path,
|
||||
findings: Sequence[Any],
|
||||
engine_result: Mapping[str, Any] | None,
|
||||
*,
|
||||
base: Delta | None = None,
|
||||
rules: Sequence[PatternRule] | None = None,
|
||||
quick_languages: Iterable[str] | None = None,
|
||||
engine_languages: Iterable[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""``static`` records for a checkout: one per (path, function, family) with a quick or engine signal. ``findings``
|
||||
are the quick findings (enabled and shadow alike), ``engine_result`` the engine's record (``findings`` with
|
||||
``site``/``family``/``rung``/``witness``, ``questions``, ``completed``, ``degradations``; ``None`` when the engine
|
||||
did not run, ``questions == 0`` is ``failed``). With ``base`` only functions enclosing a changed line are
|
||||
candidates, and ``delta.*`` compares their signals with the base's."""
|
||||
from ..plane.tasks.alerts import covers
|
||||
from ..plane.tasks.alerts import engine_languages as engine_covered
|
||||
from ..plane.tasks.alerts import quick_languages as quick_covered
|
||||
from ..semantic.facts import FactLoadError, load_facts
|
||||
|
||||
root = Path(root)
|
||||
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
||||
by_id = {rule.rule_id: rule for rule in loaded}
|
||||
quick_langs = set(quick_languages if quick_languages is not None else quick_covered())
|
||||
engine_langs = set(engine_languages if engine_languages is not None else engine_covered())
|
||||
index = SourceIndex(root)
|
||||
quick, engine = _signals(index, findings, engine_result, by_id)
|
||||
keys = sorted(set(quick) | set(engine))
|
||||
changed: dict[tuple[str, str], int] = {}
|
||||
base_quick: dict[tuple[str, str, str], list[tuple[str, str]]] = {}
|
||||
base_engine: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {}
|
||||
base_index: SourceIndex | None = None
|
||||
if base is not None:
|
||||
changed = _changed_in(index, base.changed)
|
||||
keys = [k for k in keys if (k[0], k[1]) in changed]
|
||||
base_index = SourceIndex(base.root)
|
||||
base_quick, base_engine = _signals(base_index, base.findings, base.engine_result, by_id)
|
||||
entries, file_entries = entry_names(root, {k[0] for k in keys}) if keys else (set(), set())
|
||||
callers = index.callers() if keys else {}
|
||||
quick_version = ruleset_digest(loaded)
|
||||
engine_state = _engine_state(engine_result)
|
||||
completed, questions = (engine_result or {}).get("completed"), (engine_result or {}).get("questions")
|
||||
completion = int(completed) / int(questions) if isinstance(completed, int) and isinstance(questions, int) and questions else None
|
||||
degraded = bool((engine_result or {}).get("degradations"))
|
||||
vocabularies: dict[str, Vocabulary] = {}
|
||||
try:
|
||||
sanitizer_calls = {language: {c for s in load_facts().for_language(language).sanitizers for c in s.calls} for language in LANGUAGES}
|
||||
except FactLoadError:
|
||||
sanitizer_calls = {}
|
||||
out: list[dict[str, Any]] = []
|
||||
for key in keys:
|
||||
path, function, family = key
|
||||
language = detect_language(Path(path))
|
||||
parts: dict[str, Part] = {}
|
||||
parts["quick"] = quick_part(quick.get(key, []), by_id, quick_version) if covers(quick_langs, language) else NONE
|
||||
if language in engine_langs and engine_state != "none":
|
||||
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
|
||||
parts["engine"] = (
|
||||
engine_part(engine.get(key, []), completion=completion, degraded=degraded, vocabulary=vocabulary)
|
||||
if engine_state == "ran" else Part("failed")
|
||||
) # fmt: skip
|
||||
lines = index.lines(path)
|
||||
parts["source"] = source_part(lines, language, function)
|
||||
distance = 0 if function == GLOBAL and path in file_entries else entry_distance(function, entries, callers)
|
||||
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
|
||||
parts["facts"] = facts_part(_callers_of(index, callers, path, function), function)
|
||||
if base is None or base_index is None:
|
||||
parts["delta"] = NOT_APPLICABLE
|
||||
else:
|
||||
head_n = len(quick.get(key, [])) + len(engine.get(key, []))
|
||||
base_n = len(base_quick.get(key, [])) + len(base_engine.get(key, []))
|
||||
novelty = "new" if base_n == 0 else "worsened" if head_n > base_n else "unchanged"
|
||||
known = sanitizer_calls.get(language, set())
|
||||
removed = bool((_body_calls(base_index.lines(path), language, function) - _body_calls(lines, language, function)) & known)
|
||||
values = {"delta.novelty": novelty, "delta.changed_lines": changed.get((path, function), 0), "delta.sanitizer_removed": removed}
|
||||
parts["delta"] = Part("ran", values)
|
||||
unit, pin = ("delta", base.pin) if base is not None else ("pin", None)
|
||||
out.append(record(f"{path}::{function}", family, language, "static", parts, unit=unit, base=pin))
|
||||
return out
|
||||
|
||||
|
||||
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
|
||||
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
|
||||
if function == GLOBAL or function not in callers:
|
||||
return []
|
||||
sites: list[dict[str, Any]] = []
|
||||
pattern = re.compile(r"(?<![A-Za-z0-9_$])" + re.escape(function) + r"\s*\(")
|
||||
for other in index.files():
|
||||
if other == path or detect_language(Path(other)) != detect_language(Path(path)):
|
||||
continue
|
||||
for number, line in enumerate(index.lines(other) or (), start=1):
|
||||
if pattern.search(line) and not line.lstrip().startswith(_COMMENT_PREFIXES):
|
||||
sites.append({"path": other, "line": number})
|
||||
if len(sites) >= 40:
|
||||
return sites
|
||||
return sites
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CWE_OPERATION", "NONE", "NOT_APPLICABLE", "SOURCE_KIND", "Delta", "Part", "SourceIndex", "Vocabulary", "agree_part",
|
||||
"build_for_scan", "engine_part", "entry_distance", "entry_names", "facts_part", "function_span", "language_of", "quick_part", "record",
|
||||
"ruleset_digest", "source_part", "verify_part",
|
||||
] # fmt: skip
|
||||
@@ -0,0 +1,258 @@
|
||||
"""The candidate feature record: a closed, typed allow-list (learned-decision-engine Req 1, 7.2; design section 1).
|
||||
|
||||
A candidate is ``(repo, pin, path, function, family)``. That key lives in the record's envelope, for joining labels
|
||||
and grouping folds, and is never a feature. The features are the closed list :data:`FEATURES`: each has a name, the
|
||||
instrument that produces it, a type (``bool``, ``int`` with a cap, ``float`` in [0, 1], or ``enum`` over a closed
|
||||
vocabulary), a profile (``static``: what ``scan``/``pre-push`` compute locally without a model; ``plane``: adds
|
||||
verify, agree and model roles) and whether it is a framework *prior* (Req 8.2). There is no free text, no
|
||||
identifier and no raw count of a repository-specific token: rule ids enter only through the closed mechanism
|
||||
buckets and the prior count, because a rule written after one case's miss would otherwise name that case.
|
||||
|
||||
**Missing is not negative (Req 1.2).** A record carries ``instruments: {name: {state, version}}`` with ``state`` in
|
||||
:data:`STATES`:
|
||||
|
||||
- ``ran``: the instrument read its input for this candidate; its features hold values (``0`` is a real zero);
|
||||
- ``none``: no coverage -- the instrument has no rule/model for the language, or it was not part of the run;
|
||||
- ``failed``: the instrument broke or did not demonstrably read its input (never zero findings);
|
||||
- ``not_applicable``: ``delta.*`` on a whole-repository scan, ``facts.*`` on a pair excerpt.
|
||||
|
||||
Every feature of an instrument whose state is not ``ran`` is ``null``. A feature of an instrument that ran is
|
||||
``null`` only where the spec says ``nullable`` (a minimum over no findings, an answer to a pass never asked).
|
||||
At training time a ``null`` becomes 0 plus one indicator ``missing.<instrument>`` per instrument.
|
||||
|
||||
**Excluded** (label information or identities, :data:`EXCLUDED_FIELDS`): ``site_match``, ``in_fix_range``, every
|
||||
fixed-pin alert, declared sites and ``case.json``, costs, rule ids, paths, file and function names, commit ids and
|
||||
repository names. :func:`validate_record` rejects them anywhere in a record's features, and the builders drop them on
|
||||
read.
|
||||
|
||||
Versioning: :data:`SCHEMA_VERSION` (bumped on any spec change) and :func:`feature_set_digest` (sha256 of the
|
||||
canonical :data:`FEATURES`), recorded in every record, so a model trained on one feature set never scores another.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any, Literal
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
PROFILES = ("static", "plane")
|
||||
STATES = ("ran", "none", "failed", "not_applicable")
|
||||
UNITS = ("pin", "delta")
|
||||
Profile = Literal["static", "plane"]
|
||||
FeatureType = Literal["bool", "int", "float", "enum"]
|
||||
|
||||
# The instruments, in record order. `language` always runs (it is the file's extension).
|
||||
INSTRUMENTS = (
|
||||
"language", "quick", "engine", "facts", "source", "entry_points", "roles", "delta", "verify", "agree", "model_sinks",
|
||||
) # fmt: skip
|
||||
|
||||
# The languages the engine or quick mode covers; anything else is `other` (a one-hot, never a name).
|
||||
LANGUAGES = ("c", "cpp", "groovy", "java", "javascript", "php", "python", "typescript", "other")
|
||||
|
||||
# The closed mechanism vocabulary: the ids of `benchmarks/pairs/mechanisms.toml` (a test keeps them equal).
|
||||
MECHANISMS = (
|
||||
"source_reaches_sink", "container_taint", "config_sink_weak_literal", "unchecked_length_copy", "unchecked_alloc_size",
|
||||
"null_deref_unguarded", "signed_size_operand", "fixed_buffer_capacity", "identity_from_request_body",
|
||||
"missing_auth_guard", "secret_in_client", "permissive_default", "path_join_user_input", "prototype_pollution", "redos",
|
||||
"uaf", "type_confusion", "validation_strength", "cross_artifact", "unconstrained_protected_read",
|
||||
"unconstrained_protected_write", "unguarded_privileged_action", "other",
|
||||
) # fmt: skip
|
||||
|
||||
# A quick rule's `tags` mapped into the mechanism vocabulary; an unmapped tag counts as `other`.
|
||||
RULE_TAG_MECHANISM = {
|
||||
"injection": "source_reaches_sink",
|
||||
"syscall_entry": "source_reaches_sink",
|
||||
"deserialization": "source_reaches_sink",
|
||||
"network_entry": "source_reaches_sink",
|
||||
"filesystem_entry": "path_join_user_input",
|
||||
"memory_unsafe": "unchecked_length_copy",
|
||||
"crypto": "config_sink_weak_literal",
|
||||
"insecure_default": "permissive_default",
|
||||
}
|
||||
|
||||
RUNGS = ("suspicion", "model_corroborated", "model_entailed", "execution_confirmed")
|
||||
OPERATIONS = (
|
||||
"sql", "command", "code_eval", "deserialize", "file_path", "file_inclusion", "outbound_request", "redirect", "html_output",
|
||||
"other",
|
||||
) # fmt: skip
|
||||
SOURCE_KINDS = ("request", "server", "raw_body", "argv_stdin", "inferred_role", "prior", "other")
|
||||
ORIGINS = ("language", "inferred_wrapper", "model_role", "prior")
|
||||
NOVELTY = ("new", "worsened", "unchanged", "unknown", "not_applicable")
|
||||
FINALS = ("agreed", "disputed", "rejected")
|
||||
|
||||
# Fields that carry label information or an identity. None may be a feature; builders drop them on read.
|
||||
EXCLUDED_FIELDS = frozenset(
|
||||
{
|
||||
"site_match", "in_fix_range", "fixed_ranges", "ranges", "sites", "declared_sites", "case", "case_json",
|
||||
"usd", "cost", "rule_id", "rule_ids", "path", "file", "function", "candidate", "repo", "repository", "pin",
|
||||
"commit", "base", "sha", "title", "witness", "site", "reported_at",
|
||||
}
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class FeatureRecordError(ValueError):
|
||||
"""A record names a feature outside the allow-list, a value outside its type, or a label/identity field."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeatureSpec:
|
||||
name: str
|
||||
instrument: str
|
||||
type: FeatureType
|
||||
profile: Profile
|
||||
cap: int | None = None # int: the value is clipped to [0, cap]
|
||||
vocabulary: tuple[str, ...] = () # enum: the closed set of values
|
||||
prior: bool = False # a framework/library prior (Req 8.2): off by default for the engine
|
||||
nullable: bool = False # may be null while its instrument ran (a minimum over nothing, a pass never asked)
|
||||
|
||||
|
||||
def _specs() -> tuple[FeatureSpec, ...]:
|
||||
specs: list[FeatureSpec] = [FeatureSpec(f"lang.{name}", "language", "bool", "static") for name in LANGUAGES]
|
||||
specs += [
|
||||
FeatureSpec("qr.enabled_hits", "quick", "int", "static", cap=20),
|
||||
FeatureSpec("qr.shadow_hits", "quick", "int", "static", cap=20),
|
||||
]
|
||||
specs += [FeatureSpec(f"qr.mechanism.{m}", "quick", "int", "static", cap=10) for m in MECHANISMS]
|
||||
specs += [
|
||||
FeatureSpec("qr.max_precision_estimate", "quick", "float", "static", nullable=True),
|
||||
FeatureSpec("qr.prior_hits", "quick", "int", "static", cap=10, prior=True),
|
||||
FeatureSpec("eng.findings", "engine", "int", "static", cap=10),
|
||||
FeatureSpec("eng.rung_max", "engine", "enum", "static", vocabulary=RUNGS, nullable=True),
|
||||
FeatureSpec("eng.witness_steps_min", "engine", "int", "static", cap=50, nullable=True),
|
||||
FeatureSpec("eng.sink_kind", "engine", "enum", "static", vocabulary=OPERATIONS, nullable=True),
|
||||
FeatureSpec("eng.source_kind", "engine", "enum", "static", vocabulary=SOURCE_KINDS, nullable=True),
|
||||
FeatureSpec("eng.sink_origin", "engine", "enum", "static", vocabulary=ORIGINS, nullable=True),
|
||||
FeatureSpec("eng.source_origin", "engine", "enum", "static", vocabulary=ORIGINS, nullable=True),
|
||||
FeatureSpec("eng.sanitizer_on_path", "engine", "bool", "static", nullable=True),
|
||||
FeatureSpec("eng.completion", "engine", "float", "static", nullable=True),
|
||||
FeatureSpec("eng.degraded", "engine", "bool", "static"),
|
||||
FeatureSpec("eng.unstable", "engine", "bool", "static", nullable=True),
|
||||
FeatureSpec("facts.callers", "facts", "int", "static", cap=40),
|
||||
FeatureSpec("facts.caller_files", "facts", "int", "static", cap=20),
|
||||
FeatureSpec("facts.is_global", "facts", "bool", "static"),
|
||||
FeatureSpec("facts.function_lines", "source", "int", "static", cap=500),
|
||||
FeatureSpec("facts.entry_distance", "entry_points", "int", "static", cap=6),
|
||||
FeatureSpec("roles.sink_confidence", "roles", "float", "static", nullable=True),
|
||||
FeatureSpec("roles.source_in_function", "roles", "bool", "static"),
|
||||
FeatureSpec("delta.novelty", "delta", "enum", "static", vocabulary=NOVELTY),
|
||||
FeatureSpec("delta.changed_lines", "delta", "int", "static", cap=200),
|
||||
FeatureSpec("delta.sanitizer_removed", "delta", "bool", "static"),
|
||||
FeatureSpec("verify.flag_a", "verify", "bool", "plane", nullable=True),
|
||||
FeatureSpec("verify.flag_b", "verify", "bool", "plane", nullable=True),
|
||||
FeatureSpec("verify.flag_c", "verify", "bool", "plane", nullable=True),
|
||||
FeatureSpec("verify.votes", "verify", "int", "plane", cap=3),
|
||||
FeatureSpec("verify.turns_mean", "verify", "float", "plane", nullable=True),
|
||||
FeatureSpec("verify.site_offset", "verify", "int", "plane", cap=200, nullable=True),
|
||||
FeatureSpec("agree.final", "agree", "enum", "plane", vocabulary=FINALS),
|
||||
FeatureSpec("ms.flagged", "model_sinks", "bool", "plane"),
|
||||
FeatureSpec("ms.operation", "model_sinks", "enum", "plane", vocabulary=OPERATIONS, nullable=True),
|
||||
]
|
||||
return tuple(specs)
|
||||
|
||||
|
||||
FEATURES: tuple[FeatureSpec, ...] = _specs()
|
||||
BY_NAME: Mapping[str, FeatureSpec] = {spec.name: spec for spec in FEATURES}
|
||||
# `float` features are fractions or means; `verify.turns_mean` is a mean of tool turns, the only unbounded float.
|
||||
UNBOUNDED_FLOATS = frozenset({"verify.turns_mean"})
|
||||
|
||||
|
||||
def features_for(profile: str) -> tuple[FeatureSpec, ...]:
|
||||
"""The features a record of ``profile`` carries: ``static`` never sees an instrument absent in deployment."""
|
||||
if profile not in PROFILES:
|
||||
raise FeatureRecordError(f"unknown profile {profile!r} (one of {', '.join(PROFILES)})")
|
||||
return FEATURES if profile == "plane" else tuple(spec for spec in FEATURES if spec.profile == "static")
|
||||
|
||||
|
||||
def instruments_for(profile: str) -> tuple[str, ...]:
|
||||
used = {spec.instrument for spec in features_for(profile)}
|
||||
return tuple(name for name in INSTRUMENTS if name in used)
|
||||
|
||||
|
||||
def feature_set_digest() -> str:
|
||||
"""sha256 of the canonical feature list: any change to a name, type, cap, vocabulary or profile changes it."""
|
||||
canonical = json.dumps([asdict(spec) for spec in FEATURES], sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _check_value(spec: FeatureSpec, value: object) -> None:
|
||||
where = f"feature {spec.name}"
|
||||
if spec.type == "bool":
|
||||
if not isinstance(value, bool):
|
||||
raise FeatureRecordError(f"{where}: expected a bool, got {value!r}")
|
||||
elif spec.type == "int":
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= (spec.cap or 0):
|
||||
raise FeatureRecordError(f"{where}: expected an int in [0, {spec.cap}], got {value!r}")
|
||||
elif spec.type == "float":
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
||||
raise FeatureRecordError(f"{where}: expected a finite non-negative number, got {value!r}")
|
||||
if spec.name not in UNBOUNDED_FLOATS and value > 1:
|
||||
raise FeatureRecordError(f"{where}: expected a fraction in [0, 1], got {value!r}")
|
||||
elif not isinstance(value, str) or value not in spec.vocabulary:
|
||||
raise FeatureRecordError(f"{where}: {value!r} is outside the vocabulary {list(spec.vocabulary)}")
|
||||
|
||||
|
||||
def validate_x(x: Mapping[str, object], instruments: Mapping[str, Mapping[str, object]], profile: str) -> None:
|
||||
"""The features of one record against the allow-list and the instruments' states."""
|
||||
specs = {spec.name: spec for spec in features_for(profile)}
|
||||
unknown = sorted(set(x) - set(specs))
|
||||
if unknown:
|
||||
label = sorted(k for k in unknown if k.rsplit(".", 1)[-1] in EXCLUDED_FIELDS)
|
||||
why = f" (label or identity fields: {label})" if label else ""
|
||||
raise FeatureRecordError(f"features outside the {profile} allow-list: {unknown}{why}")
|
||||
missing = sorted(set(specs) - set(x))
|
||||
if missing:
|
||||
raise FeatureRecordError(f"features missing from the record: {missing}")
|
||||
wanted = set(instruments_for(profile))
|
||||
if set(instruments) != wanted:
|
||||
raise FeatureRecordError(f"instruments {sorted(instruments)} != the {profile} profile's {sorted(wanted)}")
|
||||
for name, info in instruments.items():
|
||||
if not isinstance(info, Mapping) or info.get("state") not in STATES:
|
||||
raise FeatureRecordError(f"instrument {name}: state must be one of {STATES}, got {info!r}")
|
||||
for name, spec in specs.items():
|
||||
value, state = x[name], instruments[spec.instrument]["state"]
|
||||
if state != "ran":
|
||||
if value is not None:
|
||||
raise FeatureRecordError(f"feature {name}: instrument {spec.instrument} is {state}, so the value must be null")
|
||||
continue
|
||||
if value is None:
|
||||
if not spec.nullable:
|
||||
raise FeatureRecordError(f"feature {name}: instrument {spec.instrument} ran, so null is not allowed here")
|
||||
continue
|
||||
_check_value(spec, value)
|
||||
|
||||
|
||||
RECORD_FIELDS = ("candidate", "family", "language", "profile", "schema_version", "feature_set_digest", "unit", "x", "instruments")
|
||||
|
||||
|
||||
def validate_record(record: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""A feature record, checked: envelope fields, versions, profile, and :func:`validate_x`. Returns it as a dict."""
|
||||
extra = sorted(set(record) - {*RECORD_FIELDS, "base"})
|
||||
if extra:
|
||||
raise FeatureRecordError(f"record fields outside the schema: {extra}")
|
||||
missing = [name for name in RECORD_FIELDS if name not in record]
|
||||
if missing:
|
||||
raise FeatureRecordError(f"record fields missing: {missing}")
|
||||
if record["schema_version"] != SCHEMA_VERSION or record["feature_set_digest"] != feature_set_digest():
|
||||
raise FeatureRecordError("record built for another schema version or feature set")
|
||||
if record["unit"] not in UNITS:
|
||||
raise FeatureRecordError(f"unit {record['unit']!r} is not one of {UNITS}")
|
||||
if not isinstance(record["x"], Mapping) or not isinstance(record["instruments"], Mapping):
|
||||
raise FeatureRecordError("x and instruments must be objects")
|
||||
validate_x(record["x"], record["instruments"], str(record["profile"]))
|
||||
return dict(record)
|
||||
|
||||
|
||||
def names(specs: Iterable[FeatureSpec] = FEATURES) -> tuple[str, ...]:
|
||||
return tuple(spec.name for spec in specs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BY_NAME", "EXCLUDED_FIELDS", "FEATURES", "FINALS", "INSTRUMENTS", "LANGUAGES", "MECHANISMS", "NOVELTY", "OPERATIONS",
|
||||
"ORIGINS", "PROFILES", "RULE_TAG_MECHANISM", "RUNGS", "SCHEMA_VERSION", "SOURCE_KINDS", "STATES", "UNITS",
|
||||
"FeatureRecordError", "FeatureSpec", "feature_set_digest", "features_for", "instruments_for", "names",
|
||||
"validate_record", "validate_x",
|
||||
] # fmt: skip
|
||||
@@ -68,6 +68,7 @@ SPEC_OWNED = {
|
||||
"plane.tasks.alerts": "harnessx-removal Req 6.3: quick-mode alerts on a case's vulnerable and fixed pins, model-free",
|
||||
"plane.engine_alerts": "harnessx-removal Req 6.3: engine alerts where quick mode has no rules (PHP), on the host",
|
||||
"plane.tasks.loop": "harnessx-removal Req 6.3: the improvement loop's snapshot, measure, propose and improve steps",
|
||||
"plane.tasks.features": "learned-decision-engine Req 1.1-1.3/7.2: one allow-listed feature record per candidate, model-free",
|
||||
"plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`",
|
||||
"plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory",
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
|
||||
(``triage.json``, for the measurement);
|
||||
- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
|
||||
Workspaces and naming the files as env;
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> remember``,
|
||||
- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree -> verify c -> final -> features ->
|
||||
remember`` (``features``: one allow-listed feature record per candidate, learned-decision-engine task 1),
|
||||
with budgets, annotated with the population and split (``openultrasast.io/population``/``split``: the files'
|
||||
stems) the memory rows carry.
|
||||
|
||||
@@ -86,7 +87,7 @@ CASE_PATH = "/workspace/case"
|
||||
INPUTS_PATH = "/workspace/inputs"
|
||||
FIXED_PATH = "/workspace/fixed"
|
||||
PROJECT_PATH = "/workspace/project"
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop")
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop", "features")
|
||||
FIXED_RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, new side: the fix's lines at the fixed pin"
|
||||
LOOP_MANIFEST = "benchmarks/manifests/java-spring-boot-vulnerable.toml" # the equality baseline's `improve` manifest
|
||||
LOOP_CATALOG = "benchmarks/pairs/catalog.toml"
|
||||
@@ -302,12 +303,9 @@ def _free() -> dict[str, int]:
|
||||
return {"usd": 0, "calls": 0} # model-free: budget.py refuses any call, so a stray one fails loudly
|
||||
|
||||
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
|
||||
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
|
||||
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
def _chain_inputs(case_id: str, alerts: bool) -> dict[str, str]:
|
||||
"""Every artifact of a case's chain -- facts, the three passes with their summaries, the final agree -- and, in a
|
||||
loop, its alerts with their coverage."""
|
||||
inputs = {"facts": f"{case_id}-facts/facts.json", "facts_summary": f"{case_id}-facts/summary.json"}
|
||||
for label in ("a", "b", "c"):
|
||||
inputs.update({f"pass_{label}": f"{case_id}-v{label}/units.jsonl", f"summary_{label}": f"{case_id}-v{label}/summary.json"})
|
||||
@@ -315,6 +313,28 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str,
|
||||
if alerts:
|
||||
inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl"
|
||||
inputs["alerts_summary"] = f"{case_id}-alerts/summary.json" # the coverage: whether a zero could mean clean
|
||||
return inputs
|
||||
|
||||
|
||||
def _features(item: CaseInputs, templates: Mapping[str, Task], alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `features` (learned-decision-engine design section 1): model-free, bound to the
|
||||
vulnerable Workspace for the functions' spans and entry-point distance; the chain's artifacts as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"}
|
||||
task = _task(templates["features"], f"features-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = _chain_inputs(case_id, alerts)
|
||||
entry = {"name": f"{case_id}-features", "task": f"features-{case_id}", "inputs": inputs, "outputs": ["features.jsonl", "summary.json"]}
|
||||
return task, {**entry, "budget": _free()}
|
||||
|
||||
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
|
||||
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) and the case's
|
||||
feature records as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
|
||||
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = {**_chain_inputs(case_id, alerts), "features": f"{case_id}-features/features.jsonl"}
|
||||
entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()}
|
||||
return task, entry
|
||||
|
||||
@@ -382,7 +402,7 @@ def render(
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
|
||||
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
|
||||
case ends in `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
|
||||
case ends in `features` -> `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
|
||||
snapshot, `loop-measure`, `loop-propose` and `loop-improve` (design section 6), all model-free."""
|
||||
header = f"# generated by `{command}`; do not edit, regenerate.\n"
|
||||
out: dict[str, str] = {}
|
||||
@@ -435,6 +455,9 @@ def render(
|
||||
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
|
||||
tasks.append(alerts_task)
|
||||
entries.append(alerts_entry)
|
||||
features_task, features_entry = _features(item, templates, loop is not None)
|
||||
tasks.append(features_task)
|
||||
entries.append(features_entry)
|
||||
remember_task, remember_entry = _remember(item, templates, population, split, loop is not None)
|
||||
tasks.append(remember_task)
|
||||
entries.append(remember_entry)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""The plane's memory store (harnessx-removal Requirement 6.1/6.4, design section 4).
|
||||
|
||||
One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows
|
||||
of six kinds (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``proposal_outcome``) that ``remember``
|
||||
(:mod:`.tasks.remember`) derives from a run's delivered artifacts. Layout, identical for both backends::
|
||||
of the kinds in ``KINDS`` (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``features``, ...) that
|
||||
``remember`` (:mod:`.tasks.remember`) derives from a run's delivered artifacts, plus the loop's and the decision
|
||||
engine's own. Layout, identical for both backends::
|
||||
|
||||
index.jsonl one row per ingested (run, task): sha256 of its rows, row count
|
||||
facts/<sha256>.json facts.json by content (identical facts stored once)
|
||||
@@ -49,7 +50,12 @@ MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
|
||||
POPULATION_ANNOTATION = "openultrasast.io/population"
|
||||
SPLIT_ANNOTATION = "openultrasast.io/split"
|
||||
SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"}
|
||||
KINDS = ("facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome")
|
||||
# The decision engine's kinds (learned-decision-engine, Data Models): `features` from the `features` task through
|
||||
# `remember`; `label`, `decision`, `experiment`, `arm_outcome` and `experiment_result` for the later tasks.
|
||||
KINDS = (
|
||||
"facts", "verdict", "unit_cost", "alert", "coverage", "proposal_outcome",
|
||||
"features", "label", "decision", "experiment", "arm_outcome", "experiment_result",
|
||||
) # fmt: skip
|
||||
ROW_FIELDS = ("id", "kind", "repo", "pin", "run", "task", "population", "split", "image")
|
||||
TAG_FIELDS = ("repo", "pin", "kind", "family", "run", "population", "split")
|
||||
MIN_FREE_BYTES = 1 << 30
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""`features` (learned-decision-engine design section 1): one feature record per candidate of a case. No model.
|
||||
|
||||
Budget ``{usd: 0, calls: 0}``, one per case, after ``final`` (and ``alerts`` in a loop Run). A candidate is a
|
||||
``(path, function, family)`` that the verify passes asked about or a vulnerable-pin alert named. It reads:
|
||||
|
||||
- ``facts.json`` (repo-facts callers), the verify passes' ``units.jsonl`` a, b, c and their ``summary.json`` (the
|
||||
model id is the verify instrument's version), the final ``agreed.json`` -- with ``site_match`` dropped on read;
|
||||
- in a loop, ``alerts.jsonl`` -- vulnerable-pin rows only, ``in_fix_range`` dropped: a deployed scan has no fixed pin
|
||||
and no fix -- and the alerts' ``summary.json`` (coverage per language, the engine's questions and degradations);
|
||||
- the bound checkout (``OUSAST_WORKSPACE_DIR``) for the function's span and its entry-point distance.
|
||||
|
||||
It writes ``features.jsonl`` (records validated by :func:`..learn.schema.validate_record`, the `plane` profile) and
|
||||
``summary.json``; ``remember`` turns the records into ``features`` rows. An instrument without coverage for the
|
||||
candidate's language is ``none``, one that did not read its input ``failed``: never a zero (Req 1.2). The task fails
|
||||
when every instrument it could have used failed.
|
||||
|
||||
Env: ``OUSAST_OUTPUT_DIR``, ``OUSAST_INPUT_FACTS`` (and ``OUSAST_INPUT_FACTS_SUMMARY``), ``OUSAST_INPUT_PASS_A``/``_B``/``_C``,
|
||||
``OUSAST_INPUT_SUMMARY_A``/``_B``/``_C``, ``OUSAST_INPUT_AGREED``, optional ``OUSAST_INPUT_ALERTS`` and
|
||||
``OUSAST_INPUT_ALERTS_SUMMARY``, optional ``OUSAST_WORKSPACE_DIR``. Exit 0 done, 2 failed. On the host,
|
||||
:func:`case_features` does the same over a recorded run directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ...learn.features import (
|
||||
NONE,
|
||||
NOT_APPLICABLE,
|
||||
Part,
|
||||
SourceIndex,
|
||||
Vocabulary,
|
||||
agree_part,
|
||||
engine_part,
|
||||
entry_distance,
|
||||
entry_names,
|
||||
facts_part,
|
||||
quick_part,
|
||||
record,
|
||||
ruleset_digest,
|
||||
source_part,
|
||||
verify_part,
|
||||
)
|
||||
from ...model.taxonomy import load_families
|
||||
from ...preprocess import detect_language
|
||||
from ...ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
||||
from .alerts import covers
|
||||
from .repo_facts import GLOBAL
|
||||
|
||||
OUTPUT = "features.jsonl"
|
||||
PASSES = ("a", "b", "c")
|
||||
ENGINE_PREFIX = "engine:"
|
||||
# Read, never passed on: label information (the declared site, the fix) and the fixed pin.
|
||||
DROPPED_ON_READ = ("site_match", "in_fix_range")
|
||||
|
||||
|
||||
def _drop(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in row.items() if k not in DROPPED_ON_READ}
|
||||
|
||||
|
||||
def _family(rule: PatternRule | None) -> str:
|
||||
family = load_families().family_of_cwe(rule.cwe) if rule is not None else None
|
||||
return family.id if family is not None else "unknown"
|
||||
|
||||
|
||||
def _line(site: object) -> int | None:
|
||||
parts = str(site or "").split(":")
|
||||
return int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else None
|
||||
|
||||
|
||||
def _final(row: Mapping[str, Any], disputed: set[str]) -> str:
|
||||
if row.get("agreed"):
|
||||
return "agreed"
|
||||
return "disputed" if str(row["candidate"]) in disputed else "rejected"
|
||||
|
||||
|
||||
def _quick(
|
||||
summary: Mapping[str, Any] | None, language: str, hits: list[tuple[str, str]], rules: Mapping[str, PatternRule], version: str
|
||||
) -> Part:
|
||||
if summary is None:
|
||||
return NONE
|
||||
if summary.get("status") not in (None, "done"):
|
||||
return Part("failed")
|
||||
if not covers(summary.get("quick_languages") or (), language):
|
||||
return NONE
|
||||
return quick_part(hits, rules, version)
|
||||
|
||||
|
||||
def _engine(summary: Mapping[str, Any] | None, language: str, rows: list[dict[str, Any]], vocabulary: Vocabulary) -> Part:
|
||||
if summary is None or language not in (summary.get("engine_languages") or ()):
|
||||
return NONE
|
||||
run = (summary.get("engine") or {}).get("vulnerable")
|
||||
if not isinstance(run, Mapping):
|
||||
return NONE
|
||||
questions, completed = int(run.get("questions") or 0), int(run.get("completed") or 0)
|
||||
if questions <= 0 or int(run.get("files") or 0) <= 0:
|
||||
return Part("failed", version=summary.get("image"))
|
||||
return engine_part(
|
||||
rows, completion=completed / questions, degraded=bool(run.get("degradations")), vocabulary=vocabulary, version=summary.get("image")
|
||||
)
|
||||
|
||||
|
||||
def case_features(
|
||||
*,
|
||||
facts: Mapping[str, Any] | None,
|
||||
passes: Mapping[str, Sequence[Mapping[str, Any]]],
|
||||
models: Mapping[str, str | None],
|
||||
agreed: Mapping[str, Any] | None,
|
||||
alerts: Sequence[Mapping[str, Any]] = (),
|
||||
alerts_summary: Mapping[str, Any] | None = None,
|
||||
workspace: Path | None = None,
|
||||
rules: Sequence[PatternRule] | None = None,
|
||||
facts_summary: Mapping[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The `plane` records of one case, sorted by (candidate, family). Versions: the ruleset digest (quick), the
|
||||
engine image (engine), the image the facts were computed on (facts), the verify model ids (verify)."""
|
||||
loaded = tuple(rules) if rules is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
||||
by_id = {rule.rule_id: rule for rule in loaded}
|
||||
candidates: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
verdicts = [_drop(c) for c in (agreed or {}).get("candidates") or []]
|
||||
disputed = {str(d["candidate"]) for d in (agreed or {}).get("disputed") or []}
|
||||
for row in verdicts:
|
||||
family = str((row.get("finding") or {}).get("family") or (agreed or {}).get("family") or "unknown")
|
||||
entry = candidates.setdefault((str(row["candidate"]), family), {"line": _line(row.get("site")), "quick": [], "engine": []})
|
||||
entry["final"] = _final(row, disputed)
|
||||
for alert in (_drop(a) for a in alerts):
|
||||
if alert.get("pin_role") != "vulnerable":
|
||||
continue # a deployed scan has no fixed pin
|
||||
rule_id = str(alert.get("rule_id") or "")
|
||||
engine = rule_id.startswith(ENGINE_PREFIX)
|
||||
family = rule_id.removeprefix(ENGINE_PREFIX) if engine else _family(by_id.get(rule_id))
|
||||
key = (f"{alert.get('path')}::{alert.get('function') or GLOBAL}", family)
|
||||
entry = candidates.setdefault(key, {"line": alert.get("line"), "quick": [], "engine": []})
|
||||
if engine:
|
||||
entry["engine"].append({"site": f"{alert.get('path')}:{alert.get('line')}"})
|
||||
else:
|
||||
entry["quick"].append((rule_id, str(alert.get("rule_status") or "enabled")))
|
||||
callers = (facts or {}).get("callers") or {}
|
||||
index = SourceIndex(workspace) if workspace is not None and workspace.is_dir() else None
|
||||
entries: tuple[set[str], set[str]] | None = None
|
||||
quick_version = ruleset_digest(loaded)
|
||||
vocabularies: dict[str, Vocabulary] = {}
|
||||
reused = (facts_summary or {}).get("reused")
|
||||
facts_image = (reused.get("image") if isinstance(reused, Mapping) else None) or (facts_summary or {}).get("image")
|
||||
facts_version = str(facts_image) if facts_image else None
|
||||
verify_version = "+".join(sorted({m for m in models.values() if m})) or None
|
||||
out: list[dict[str, Any]] = []
|
||||
for (candidate, family), entry in sorted(candidates.items()):
|
||||
path, _, function = candidate.partition("::")
|
||||
language = detect_language(Path(path))
|
||||
vocabulary = vocabularies.setdefault(language, Vocabulary.load(language))
|
||||
parts: dict[str, Part] = {
|
||||
"quick": _quick(alerts_summary, language, entry["quick"], by_id, quick_version),
|
||||
"engine": _engine(alerts_summary, language, entry["engine"], vocabulary),
|
||||
"facts": facts_part(callers[candidate], function, facts_version) if candidate in callers else NONE,
|
||||
"delta": NOT_APPLICABLE,
|
||||
"verify": verify_part(passes, candidate, entry["line"], verify_version),
|
||||
"agree": agree_part(entry.get("final")),
|
||||
}
|
||||
if index is not None:
|
||||
if entries is None:
|
||||
entries = entry_names(index.root, {c.partition("::")[0] for c, _ in candidates})
|
||||
parts["source"] = source_part(index.lines(path), language, function)
|
||||
distance = 0 if function == GLOBAL and path in entries[1] else entry_distance(function, entries[0], index.callers())
|
||||
parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
|
||||
out.append(record(candidate, family, language, "plane", parts))
|
||||
return out
|
||||
|
||||
|
||||
def _jsonl(path: Path | None) -> list[dict[str, Any]]:
|
||||
if path is None or not path.is_file():
|
||||
return []
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _json(path: Path | None) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path is not None and path.is_file() else None
|
||||
|
||||
|
||||
def _model(summary: object) -> str | None:
|
||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||
|
||||
|
||||
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
|
||||
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
|
||||
final = base / f"{case}-final" / "agreed.json"
|
||||
return case_features(
|
||||
facts=_json(base / f"{case}-facts" / "facts.json"),
|
||||
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
|
||||
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
|
||||
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
|
||||
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
|
||||
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
|
||||
)
|
||||
|
||||
|
||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
|
||||
def _units(path: Path | None) -> list[dict[str, Any]]:
|
||||
return _jsonl(path / "units.jsonl" if path is not None and path.is_dir() else path)
|
||||
|
||||
|
||||
def _states(records: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, int]]:
|
||||
counts: dict[str, dict[str, int]] = {}
|
||||
for rec in records:
|
||||
for name, info in rec["instruments"].items():
|
||||
counts.setdefault(name, {}).setdefault(info["state"], 0)
|
||||
counts[name][info["state"]] += 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
env = environ if environ is not None else os.environ
|
||||
output_dir = _path(env, "OUSAST_OUTPUT_DIR")
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
try:
|
||||
if output_dir is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
workspace = _path(env, "OUSAST_WORKSPACE_DIR")
|
||||
records = case_features(
|
||||
facts=_json(_path(env, "OUSAST_INPUT_FACTS")),
|
||||
passes={p: _units(_path(env, f"OUSAST_INPUT_PASS_{p.upper()}")) for p in PASSES},
|
||||
models={p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES},
|
||||
agreed=_json(_path(env, "OUSAST_INPUT_AGREED")),
|
||||
alerts=_jsonl(_path(env, "OUSAST_INPUT_ALERTS")),
|
||||
alerts_summary=_json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY")),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(_path(env, "OUSAST_INPUT_FACTS_SUMMARY")),
|
||||
)
|
||||
states = _states(records)
|
||||
used = {name: n for name, n in states.items() if name not in ("language", "delta") and set(n) - {"none"}}
|
||||
if records and used and all(set(n) == {"failed"} for n in used.values()):
|
||||
raise ValueError(f"every instrument failed for this case: {used}")
|
||||
(output_dir / OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in records), encoding="utf-8")
|
||||
summary.update(status="done", units_done=1, records=len(records), instruments=states)
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
if output_dir is not None:
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: summary.get(k) for k in ("status", "records")}), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
No model, budget ``{usd: 0, calls: 0}``, one per case. It reads the case's `facts.json`, the `units.jsonl` of the
|
||||
verify passes a, b and c, the final `agreed.json` (its per-candidate rows and `disputed` list) and, when present,
|
||||
`alerts.jsonl` with the alerts' `summary.json`, and writes `memory.jsonl` -- rows of the kinds `facts`, `verdict`,
|
||||
`unit_cost`, `alert` and `coverage` (per language and pin: whether any rule could have fired there; the kind
|
||||
`alerts.jsonl` with the alerts' `summary.json`, and the `features` task's `features.jsonl`, and writes `memory.jsonl`
|
||||
-- rows of the kinds `facts`, `verdict`, `unit_cost`, `alert`, `coverage` (per language and pin: whether any rule
|
||||
could have fired there) and `features` (one per candidate and family, validated against the allow-list; the kind
|
||||
`proposal_outcome` comes from the loop) -- and passes `facts.json` through. Task code interprets the
|
||||
artifacts, so the reconciler and the runner stay content-blind (ai-service-plane Req 3.4).
|
||||
|
||||
@@ -14,7 +15,7 @@ Env contract in a Run: `OUSAST_OUTPUT_DIR`, `OUSAST_RUN`, `OUSAST_TASK`, `OUSAST
|
||||
whose reused or recorded `image` the facts row carries; without one named, the row is omitted),
|
||||
`OUSAST_INPUT_PASS_A`/`_B`, optional `OUSAST_INPUT_PASS_C`, `OUSAST_INPUT_SUMMARY_A`/`_B`/`_C` (the passes'
|
||||
`summary.json`, for the model), `OUSAST_INPUT_AGREED`, `OUSAST_INPUT_ALERTS` (and `OUSAST_INPUT_ALERTS_SUMMARY`,
|
||||
the coverage); the repository and pin from the bound
|
||||
the coverage), `OUSAST_INPUT_FEATURES`; the repository and pin from the bound
|
||||
Workspace (`AX_WORKSPACES_YAML`, `OUSAST_GIT_PINS`), the image from `AX_TASK_YAML`, the population and split from
|
||||
`OUSAST_POPULATION`/`OUSAST_SPLIT` (the generator copies the Run's `openultrasast.io/population` and
|
||||
`openultrasast.io/split` annotations). Exit 0 done, 2 failed.
|
||||
@@ -38,6 +39,7 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ...learn.schema import validate_record
|
||||
from ..memory import REMEMBER_OUTPUT, IngestResult, MemoryStore, image_digest, ingest, repo_key, row_id, validate_row
|
||||
from .repo_facts import _digest
|
||||
|
||||
@@ -105,8 +107,10 @@ def rows_for(
|
||||
agreed: Mapping[str, Any] | None,
|
||||
alerts: Sequence[Mapping[str, Any]] = (),
|
||||
alerts_summary: Mapping[str, Any] | None = None,
|
||||
features: Sequence[Mapping[str, Any]] = (),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The memory rows of one case: facts, verdicts, unit costs, alerts, and the alerts' coverage per language and pin."""
|
||||
"""The memory rows of one case: facts, verdicts, unit costs, alerts, the alerts' coverage per language and pin, and
|
||||
the feature records (re-validated: a delivered record outside the allow-list fails the case)."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
if facts is not None:
|
||||
loaded = json.loads(facts)
|
||||
@@ -143,6 +147,9 @@ def rows_for(
|
||||
subject = f"{fields['rule_id']}:{fields['path']}:{fields['line']}" + (":fixed" if fields.get("pin_role") == "fixed" else "")
|
||||
rows.append(ctx.row("alert", subject, **fields))
|
||||
rows.extend(coverage_rows(ctx, alerts_summary))
|
||||
for rec in features:
|
||||
checked = validate_record(rec)
|
||||
rows.append(ctx.row("features", f"{checked['candidate']}:{checked['family']}", **checked))
|
||||
return [validate_row(r, f"{ctx.task} {r['kind']}") for r in rows]
|
||||
|
||||
|
||||
@@ -212,7 +219,10 @@ def case_rows(base: Path, case: str, *, population: str, split: str) -> tuple[li
|
||||
models = {p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES}
|
||||
alerts = _jsonl(base / f"{case}-alerts" / "alerts.jsonl")
|
||||
alerts_summary = _json(base / f"{case}-alerts" / "summary.json")
|
||||
rows = rows_for(ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
|
||||
features = _jsonl(base / f"{case}-features" / "features.jsonl")
|
||||
rows = rows_for(
|
||||
ctx, facts=facts, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary, features=features
|
||||
)
|
||||
return rows, facts
|
||||
|
||||
|
||||
@@ -275,7 +285,10 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
models = {p: _model(_json(_path(env, f"OUSAST_INPUT_SUMMARY_{p.upper()}"))) for p in PASSES}
|
||||
agreed = _json(_path(env, "OUSAST_INPUT_AGREED"))
|
||||
alerts, alerts_summary = _jsonl(_path(env, "OUSAST_INPUT_ALERTS")), _json(_path(env, "OUSAST_INPUT_ALERTS_SUMMARY"))
|
||||
rows = rows_for(ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary)
|
||||
features = _jsonl(_path(env, "OUSAST_INPUT_FEATURES"))
|
||||
rows = rows_for(
|
||||
ctx, facts=None, passes=passes, models=models, agreed=agreed, alerts=alerts, alerts_summary=alerts_summary, features=features
|
||||
)
|
||||
facts_image = _facts_image(env, ctx.image)
|
||||
if facts is not None and facts_image is not None: # the facts row carries the image the facts were computed on
|
||||
facts_ctx = Context(ctx.run, ctx.task, ctx.repo, ctx.pin, facts_image, ctx.population, ctx.split)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""The candidate feature schema (learned-decision-engine task 1; Req 1.2, 7.2): a closed allow-list with no
|
||||
identities, missing distinct from zero, and no label-carrying field reaching a feature."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.findings import quick_scan_findings
|
||||
from openultrasast.learn import schema
|
||||
from openultrasast.learn.features import Delta, Part, build_for_scan, record
|
||||
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
|
||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
||||
from openultrasast.plane.tasks.features import case_features
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.rank import rank_targets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP = """import os
|
||||
|
||||
|
||||
def {inner}(value):
|
||||
os.system(value)
|
||||
|
||||
|
||||
def {outer}():
|
||||
{inner}("x")
|
||||
"""
|
||||
|
||||
|
||||
def _scan(root: Path) -> list[dict[str, Any]]:
|
||||
_, targets = preprocess_repository(root)
|
||||
assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
assert findings, "the fixture's os.system fires a quick rule"
|
||||
engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
|
||||
return build_for_scan(root, findings, engine)
|
||||
|
||||
|
||||
def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_text(APP.format(inner=inner, outer=outer))
|
||||
return root
|
||||
|
||||
|
||||
# --- the allow-list ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_features_are_allow_listed(tmp_path: Path) -> None:
|
||||
names = [spec.name for spec in FEATURES]
|
||||
assert len(names) == len(set(names)), "a feature name is declared twice"
|
||||
for spec in FEATURES:
|
||||
assert spec.type in ("bool", "int", "float", "enum"), f"{spec.name}: no free text"
|
||||
assert spec.instrument in schema.INSTRUMENTS and spec.profile in schema.PROFILES
|
||||
assert (spec.type == "int") == (spec.cap is not None), spec.name
|
||||
assert (spec.type == "enum") == bool(spec.vocabulary), spec.name
|
||||
assert spec.name.rsplit(".", 1)[-1] not in EXCLUDED_FIELDS, f"{spec.name} names a label or identity field"
|
||||
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
|
||||
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
|
||||
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
|
||||
records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
|
||||
assert records
|
||||
for rec in records:
|
||||
assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
|
||||
assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
|
||||
assert {s.name for s in schema.features_for("static")} < set(names)
|
||||
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
|
||||
|
||||
|
||||
def test_record_is_name_invariant(tmp_path: Path) -> None:
|
||||
"""Two repositories that differ only in repository name, paths, file and function names and commit ids give the
|
||||
same features: nothing identifies a repository."""
|
||||
first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
|
||||
second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
|
||||
assert [r["x"] for r in first] == [r["x"] for r in second]
|
||||
assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
|
||||
assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
|
||||
assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
|
||||
root = tmp_path / "acme-shop"
|
||||
_, targets = preprocess_repository(root)
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
changed = {"app/one.py": [(5, 5)]}
|
||||
one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
|
||||
two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
|
||||
assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
|
||||
assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
|
||||
assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"
|
||||
|
||||
|
||||
# --- label fields ------------------------------------------------------------------------------------------------
|
||||
|
||||
AGREED = {
|
||||
"family": "injection",
|
||||
"candidates": [
|
||||
{"candidate": "a.py::f", "site": "a.py:4:f", "a": True, "b": False, "c": True, "agreed": True, "site_match": True,
|
||||
"finding": {"family": "injection", "reported_at": "a.py:5"}},
|
||||
],
|
||||
"disputed": [],
|
||||
} # fmt: skip
|
||||
PASSES = {
|
||||
"a": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:5"}], "turns": 4}],
|
||||
"b": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [], "turns": 6}],
|
||||
"c": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:4"}], "turns": 5}],
|
||||
}
|
||||
ALERTS = [
|
||||
{"rule_id": "python-os-command", "rule_status": "enabled", "path": "a.py", "line": 5, "function": "f", "pin_role": "vulnerable",
|
||||
"in_fix_range": True},
|
||||
] # fmt: skip
|
||||
SUMMARY = {"status": "done", "quick_languages": ["python"], "engine_languages": ["php"]}
|
||||
|
||||
|
||||
def _plane(**overrides: Any) -> list[dict[str, Any]]:
|
||||
inputs: dict[str, Any] = {
|
||||
"facts": {"callers": {"a.py::f": [{"path": "b.py", "line": 2}]}}, "passes": PASSES, "models": {"a": "m", "b": "m", "c": "m"},
|
||||
"agreed": AGREED, "alerts": ALERTS, "alerts_summary": SUMMARY,
|
||||
} # fmt: skip
|
||||
return case_features(**{**inputs, **overrides})
|
||||
|
||||
|
||||
def test_label_fields_never_reach_features() -> None:
|
||||
baseline = _plane()
|
||||
assert [r["candidate"] for r in baseline] == ["a.py::f"] and baseline[0]["profile"] == "plane"
|
||||
x = baseline[0]["x"]
|
||||
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, False, True, 2)
|
||||
assert x["agree.final"] == "agreed" and x["qr.enabled_hits"] == 1 and x["verify.site_offset"] == 0
|
||||
flipped = copy.deepcopy(AGREED)
|
||||
flipped["candidates"][0]["site_match"] = False
|
||||
outside = [{**ALERTS[0], "in_fix_range": False}]
|
||||
fixed_pin = [*ALERTS, {**ALERTS[0], "pin_role": "fixed", "line": 9}, {**ALERTS[0], "pin_role": "fixed", "function": "g"}]
|
||||
for variant in (_plane(agreed=flipped), _plane(alerts=outside), _plane(alerts=fixed_pin)):
|
||||
assert variant == baseline, "site_match, in_fix_range and fixed-pin alerts change nothing"
|
||||
for field in ("site_match", "in_fix_range", "rule_id", "path", "repo", "usd"):
|
||||
smuggled = copy.deepcopy(baseline[0])
|
||||
smuggled["x"][field] = True
|
||||
with pytest.raises(FeatureRecordError, match="allow-list"):
|
||||
validate_record(smuggled)
|
||||
with pytest.raises(FeatureRecordError):
|
||||
validate_record({**baseline[0], field: True})
|
||||
|
||||
|
||||
# --- missing is not zero -----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_is_not_zero() -> None:
|
||||
uncovered = _plane(alerts=[], alerts_summary={**SUMMARY, "quick_languages": ["php"]})[0]
|
||||
assert uncovered["instruments"]["quick"]["state"] == "none"
|
||||
assert all(v is None for k, v in uncovered["x"].items() if k.startswith("qr."))
|
||||
covered = _plane(alerts=[])[0]
|
||||
assert covered["instruments"]["quick"]["state"] == "ran"
|
||||
assert covered["x"]["qr.enabled_hits"] == 0 and covered["x"]["qr.shadow_hits"] == 0, "ran and found nothing is 0"
|
||||
no_alerts = _plane(alerts=[], alerts_summary=None)[0]
|
||||
assert no_alerts["instruments"]["quick"]["state"] == "none" and no_alerts["x"]["qr.enabled_hits"] is None
|
||||
assert covered["instruments"]["engine"]["state"] == "none" and covered["x"]["eng.findings"] is None
|
||||
assert covered["instruments"]["delta"]["state"] == "not_applicable" and covered["x"]["delta.novelty"] is None
|
||||
failed = _plane(alerts_summary={**SUMMARY, "status": "failed"})[0]
|
||||
assert failed["instruments"]["quick"]["state"] == "failed" and failed["x"]["qr.enabled_hits"] is None
|
||||
php = {"path": "p.php", "line": 3, "function": "q", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled"}
|
||||
engine = {**SUMMARY, "engine": {"vulnerable": {"questions": 0, "completed": 0, "files": 5, "degradations": []}}}
|
||||
[_, unread] = _plane(alerts=[*ALERTS, php], alerts_summary=engine)
|
||||
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
|
||||
|
||||
|
||||
def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
|
||||
root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
|
||||
_, targets = preprocess_repository(root)
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
|
||||
absent = build_for_scan(root, findings, None)
|
||||
assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
|
||||
assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
|
||||
assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))
|
||||
|
||||
|
||||
def test_validate_record_refuses_values_outside_their_type() -> None:
|
||||
good = _plane()[0]
|
||||
validate_record(good)
|
||||
for name, value in (("agree.final", "maybe"), ("verify.votes", 4), ("qr.enabled_hits", -1), ("lang.python", 1)):
|
||||
bad = copy.deepcopy(good)
|
||||
bad["x"][name] = value
|
||||
with pytest.raises(FeatureRecordError, match=name.replace(".", r"\.")):
|
||||
validate_record(bad)
|
||||
null = copy.deepcopy(good)
|
||||
null["x"]["agree.final"] = None
|
||||
with pytest.raises(FeatureRecordError, match="null is not allowed"):
|
||||
validate_record(null)
|
||||
value = copy.deepcopy(good)
|
||||
value["x"]["ms.flagged"] = False
|
||||
with pytest.raises(FeatureRecordError, match="must be null"):
|
||||
validate_record(value)
|
||||
with pytest.raises(FeatureRecordError, match="outside the static profile"):
|
||||
record("a.py::f", "injection", "python", "static", {"verify": Part("ran", {})})
|
||||
@@ -0,0 +1,83 @@
|
||||
"""The plane task `features` (learned-decision-engine task 1) on validation-46's recorded lmdeploy case
|
||||
(``tests/fixtures/plane-remember``), and `remember` storing its records as `features` rows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learn.schema import FeatureRecordError, feature_set_digest
|
||||
from openultrasast.plane.memory import FileStore
|
||||
from openultrasast.plane.tasks import features
|
||||
from openultrasast.plane.tasks.remember import Context, remember_run, rows_for
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||
CASE = "lmdeploy-media-url-ssrf"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "plane" / "validation-46"
|
||||
shutil.copytree(FIXTURE, target)
|
||||
return target
|
||||
|
||||
|
||||
def _checkout(root: Path) -> Path:
|
||||
media = root / "lmdeploy" / "vl" / "media"
|
||||
media.mkdir(parents=True)
|
||||
(media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n")
|
||||
return root
|
||||
|
||||
|
||||
def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None:
|
||||
[rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout"))
|
||||
assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == (
|
||||
"lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin",
|
||||
) # fmt: skip
|
||||
x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()}
|
||||
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2)
|
||||
assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False
|
||||
assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0
|
||||
assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran"
|
||||
assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits"
|
||||
assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest()
|
||||
assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read"
|
||||
[without] = features.recorded_case(run_dir, CASE)
|
||||
assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None
|
||||
|
||||
|
||||
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"),
|
||||
"OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"),
|
||||
**{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"},
|
||||
**{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"},
|
||||
} # fmt: skip
|
||||
assert features.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["records"], summary["usd"], summary["calls"]) == ("done", 1, 0, 0)
|
||||
assert summary["instruments"]["verify"] == {"ran": 1} and summary["instruments"]["quick"] == {"none": 1}
|
||||
(run_dir / f"{CASE}-features").mkdir()
|
||||
shutil.copy(out / features.OUTPUT, run_dir / f"{CASE}-features" / features.OUTPUT)
|
||||
store = FileStore(tmp_path / "memory")
|
||||
[result] = remember_run(run_dir, store, population="population-v2", split="validation")
|
||||
assert result.kinds == {"facts": 1, "unit_cost": 2, "verdict": 1, "features": 1}
|
||||
[row] = store.rows(kind="features")
|
||||
assert row.row["candidate"] == "lmdeploy/vl/media/connection.py::_load_http_url" and row.row["family"] == "untrusted_destination"
|
||||
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
|
||||
|
||||
|
||||
def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None:
|
||||
[rec] = features.recorded_case(run_dir, CASE)
|
||||
rec["x"]["site_match"] = True
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
with pytest.raises(FeatureRecordError, match="site_match"):
|
||||
rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec])
|
||||
|
||||
|
||||
def test_the_task_fails_loudly_without_an_output_dir() -> None:
|
||||
assert features.main({}) == 2
|
||||
@@ -56,7 +56,8 @@ def test_model_prices_are_the_endpoint_prices() -> None:
|
||||
|
||||
def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
|
||||
run, manifests = reconciler.load_run(RUN)
|
||||
assert len(run.tasks) == 105 # per case: facts, va, vb, agree, then vc, final, then remember (harnessx-removal task 4)
|
||||
# per case: facts, va, vb, agree, then vc, final, then features (learned-decision-engine task 1) and remember
|
||||
assert len(run.tasks) == 120
|
||||
population = {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]}
|
||||
ax_names: set[str] = set()
|
||||
for task in manifests.tasks.values():
|
||||
@@ -96,7 +97,9 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical(
|
||||
first = text.split(" - name: budibase-mongo-template-nosqli-vc\n", 1)[0]
|
||||
assert hashlib.sha256(first.encode()).hexdigest() == FIRST_RUN_ENTRIES
|
||||
run, manifests = reconciler.load_run(RUN)
|
||||
old, new, remembered = run.tasks[:60], run.tasks[60:90], run.tasks[90:]
|
||||
old, new, appended = run.tasks[:60], run.tasks[60:90], run.tasks[90:]
|
||||
assert [e.name.rsplit("-", 1)[1] for e in appended] == ["features", "remember"] * 15
|
||||
remembered = appended[1::2]
|
||||
assert [e.name.rsplit("-", 1)[1] for e in old] == ["facts", "va", "vb", "agree"] * 15
|
||||
assert [e.name.rsplit("-", 1)[1] for e in new] == ["vc", "final"] * 15
|
||||
for vc, final in zip(new[::2], new[1::2], strict=True):
|
||||
@@ -112,9 +115,15 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical(
|
||||
for entry in (vc, final):
|
||||
name = reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)
|
||||
assert len(name) <= reconciler.TASK_NAME_LIMIT
|
||||
for entry in appended[::2]: # learned-decision-engine task 1: model-free, the chain's artifacts in, records out
|
||||
case = entry.name.removesuffix("-features")
|
||||
assert entry.inputs["agreed"] == f"{case}-final/agreed.json" and entry.outputs == ("features.jsonl", "summary.json")
|
||||
assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0)
|
||||
assert reconciler.MODEL_ANNOTATION not in manifests.tasks[entry.task].metadata.annotations
|
||||
for entry in remembered: # harnessx-removal task 4: appended after the tie-break, model-free
|
||||
case = entry.name.removesuffix("-remember")
|
||||
assert entry.inputs["agreed"] == f"{case}-final/agreed.json" and entry.inputs["pass_c"] == f"{case}-vc/units.jsonl"
|
||||
assert entry.inputs["features"] == f"{case}-features/features.jsonl"
|
||||
assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0)
|
||||
assert reconciler.MODEL_ANNOTATION not in manifests.tasks[entry.task].metadata.annotations
|
||||
assert len(reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)) <= reconciler.TASK_NAME_LIMIT
|
||||
|
||||
@@ -169,15 +169,18 @@ def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
|
||||
def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path) -> None:
|
||||
plain = render([_case()], _templates(), "validation-46", "t", population="population-v2", split="validation")
|
||||
run, _ = reconciler.load_run(_write(tmp_path / "plain", plain))
|
||||
assert [e.name.removeprefix(f"{CASE}-") for e in run.tasks] == [*CHAIN[:4], "vc", "final", "remember"]
|
||||
assert [e.name.removeprefix(f"{CASE}-") for e in run.tasks] == [*CHAIN[:4], "vc", "final", "features", "remember"]
|
||||
remember = run.task(f"{CASE}-remember")
|
||||
assert remember.inputs["agreed"] == f"{CASE}-final/agreed.json" and "alerts" not in remember.inputs
|
||||
assert set(remember.producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final")}
|
||||
assert set(remember.producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final", "features")}
|
||||
assert set(run.task(f"{CASE}-features").producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final")}
|
||||
|
||||
files = render([_case(fixed_ranges={"x.py": [(1, 2)]})], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP)
|
||||
run, manifests = reconciler.load_run(_write(tmp_path / "loop", files))
|
||||
names = [e.name for e in run.tasks]
|
||||
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS]
|
||||
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN]
|
||||
assert names[6:] == [f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", *LOOP_TASKS]
|
||||
assert run.task(f"{CASE}-features").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl", "features read the alerts"
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl"
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too"
|
||||
measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"}
|
||||
@@ -262,6 +265,7 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
fake.script({
|
||||
f"{CASE}-alerts": {"exec": {"env": {"PYTHONPATH": SRC, "OUSAST_WORKSPACE_DIR": str(vulnerable), "OUSAST_FIXED_DIR": str(fixed),
|
||||
"OUSAST_INPUT_CASE": str(tmp_path / "case.json")}}},
|
||||
f"{CASE}-features": {"exec": {"env": {"PYTHONPATH": SRC, "OUSAST_WORKSPACE_DIR": str(vulnerable)}}},
|
||||
f"{CASE}-remember": {"exec": {"env": {"PYTHONPATH": SRC}}},
|
||||
"loop-measure": {"exec": {"env": {"PYTHONPATH": SRC}}},
|
||||
"loop-propose": {"exec": {"env": project_env}},
|
||||
@@ -271,10 +275,10 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
assert cli.main(["plane", "run", str(manifest), "--ax", str(fake.ax)]) == 0, (fake.root / "ax.log").read_text()[-3000:]
|
||||
|
||||
statuses = {name: info["status"] for name, info in json.loads((base / "state.json").read_text())["tasks"].items()}
|
||||
assert set(statuses.values()) == {"done"} and len(statuses) == 12
|
||||
assert set(statuses.values()) == {"done"} and len(statuses) == 13
|
||||
assert json.loads((base / "memory-snapshot" / "index.json").read_text())["kept"] == 4, "seeded on the host, not run"
|
||||
delivered = [name for _, name in fake.events("delivered")]
|
||||
assert delivered == [f"{CASE}-alerts", f"{CASE}-remember", "loop-measure", "loop-propose", "loop-improve"]
|
||||
assert delivered == [f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", "loop-measure", "loop-propose", "loop-improve"]
|
||||
|
||||
measure = json.loads((base / "loop-measure" / "measure.json").read_text())
|
||||
assert measure["run_metrics"]["declared_sites_agreed"] == 1 and measure["run_metrics"]["candidates"] == 1
|
||||
@@ -297,10 +301,13 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
|
||||
report = reconciler.attribution("validation-46")
|
||||
for row in report["tasks"]:
|
||||
if row["task"] in (f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS):
|
||||
if row["task"] in (f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", *LOOP_TASKS):
|
||||
assert row["model"] is None and row["usd"] in (0, None) and row["calls"] in (0, None), row
|
||||
ingested = FileStore(store.root)
|
||||
outcomes = ingested.rows(kind="proposal_outcome")
|
||||
assert {r.row["outcome"] for r in outcomes} == {"reverted"} and len(outcomes) == 2, "one per (repo, pin) of the evidence"
|
||||
[fixed_alert] = ingested.rows(repo="example.com/reserved/lmdeploy", pin=FIXED, kind="alert")
|
||||
features = {r.row["candidate"]: r.row for r in ingested.rows(kind="features")}
|
||||
assert set(features) == {"lmdeploy/vl/media/connection.py::_load_http_url"}, "no fixed-pin alert became a candidate"
|
||||
assert all(r["pin"] == VULNERABLE and "in_fix_range" not in json.dumps(r["x"]) for r in features.values())
|
||||
assert fixed_alert.row["in_fix_range"] is True and fixed_alert.row["function"] == "load"
|
||||
|
||||
Reference in New Issue
Block a user