mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
merge(learn): features schema and framework priors (decision-engine tasks 1-2)
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
@@ -11,7 +11,7 @@ def test_layout_rows_load_and_match() -> None:
|
||||
from openultrasast.model.layout import is_test_path, is_vendored, layout_facts
|
||||
|
||||
rows = layout_facts("php")
|
||||
assert [r.id for r in rows] == ["php_composer_and_wordpress"]
|
||||
assert [r.id for r in rows] == ["php_layout"]
|
||||
assert is_vendored("includes/vendor/whichbrowser/parser/data/profiles.php", rows)
|
||||
assert is_vendored("vendor/autoload.php", rows)
|
||||
assert not is_vendored("classes/class.mail.php", rows)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""The candidate feature schema (learned-decision-engine task 1; Req 1.2, 7.2): a closed allow-list with no
|
||||
identities, missing distinct from zero, and no label-carrying field reaching a feature."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.findings import quick_scan_findings
|
||||
from openultrasast.learn import schema
|
||||
from openultrasast.learn.features import Delta, Part, build_for_scan, record
|
||||
from openultrasast.learn.schema import EXCLUDED_FIELDS, FEATURES, FeatureRecordError, validate_record
|
||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
||||
from openultrasast.plane.tasks.features import case_features
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.rank import rank_targets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP = """import os
|
||||
|
||||
|
||||
def {inner}(value):
|
||||
os.system(value)
|
||||
|
||||
|
||||
def {outer}():
|
||||
{inner}("x")
|
||||
"""
|
||||
|
||||
|
||||
def _scan(root: Path) -> list[dict[str, Any]]:
|
||||
_, targets = preprocess_repository(root)
|
||||
assert targets and all((root / t.path).stat().st_size > 0 for t in targets), "the fixture was read"
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
assert findings, "the fixture's os.system fires a quick rule"
|
||||
engine = {"questions": 3, "completed": 3, "degradations": [], "findings": []}
|
||||
return build_for_scan(root, findings, engine)
|
||||
|
||||
|
||||
def _repo(root: Path, path: str, inner: str, outer: str) -> Path:
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_text(APP.format(inner=inner, outer=outer))
|
||||
return root
|
||||
|
||||
|
||||
# --- the allow-list ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_features_are_allow_listed(tmp_path: Path) -> None:
|
||||
names = [spec.name for spec in FEATURES]
|
||||
assert len(names) == len(set(names)), "a feature name is declared twice"
|
||||
for spec in FEATURES:
|
||||
assert spec.type in ("bool", "int", "float", "enum"), f"{spec.name}: no free text"
|
||||
assert spec.instrument in schema.INSTRUMENTS and spec.profile in schema.PROFILES
|
||||
assert (spec.type == "int") == (spec.cap is not None), spec.name
|
||||
assert (spec.type == "enum") == bool(spec.vocabulary), spec.name
|
||||
assert spec.name.rsplit(".", 1)[-1] not in EXCLUDED_FIELDS, f"{spec.name} names a label or identity field"
|
||||
mechanisms = tomllib.loads((ROOT / "benchmarks" / "pairs" / "mechanisms.toml").read_text())["mechanism"]
|
||||
assert tuple(m["id"] for m in mechanisms) == schema.MECHANISMS, "the mechanism buckets are the closed vocabulary"
|
||||
assert set(schema.RULE_TAG_MECHANISM.values()) <= set(schema.MECHANISMS)
|
||||
records = _scan(_repo(tmp_path, "app/one.py", "alpha", "beta"))
|
||||
assert records
|
||||
for rec in records:
|
||||
assert set(rec["x"]) == {s.name for s in schema.features_for("static")}, "every emitted key is in FEATURES"
|
||||
assert rec["profile"] == "static" and not any(k.startswith(("verify.", "agree.", "ms.")) for k in rec["x"])
|
||||
assert {s.name for s in schema.features_for("static")} < set(names)
|
||||
assert [s.name for s in FEATURES if s.prior] == ["qr.prior_hits"]
|
||||
|
||||
|
||||
def test_record_is_name_invariant(tmp_path: Path) -> None:
|
||||
"""Two repositories that differ only in repository name, paths, file and function names and commit ids give the
|
||||
same features: nothing identifies a repository."""
|
||||
first = _scan(_repo(tmp_path / "acme-shop", "app/one.py", "alpha", "beta"))
|
||||
second = _scan(_repo(tmp_path / "other-thing", "lib/sub/two.py", "gamma", "delta"))
|
||||
assert [r["x"] for r in first] == [r["x"] for r in second]
|
||||
assert [r["instruments"] for r in first] == [r["instruments"] for r in second]
|
||||
assert first[0]["candidate"] != second[0]["candidate"], "the key differs; it lives only in the envelope"
|
||||
assert first[0]["x"]["qr.enabled_hits"] >= 1 and first[0]["x"]["facts.entry_distance"] is not None
|
||||
root = tmp_path / "acme-shop"
|
||||
_, targets = preprocess_repository(root)
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
changed = {"app/one.py": [(5, 5)]}
|
||||
one = build_for_scan(root, findings, None, base=Delta(root, changed, pin="a" * 40))
|
||||
two = build_for_scan(root, findings, None, base=Delta(root, changed, pin="b" * 40))
|
||||
assert len(one) == 1 and one[0]["unit"] == "delta" and one[0]["base"] == "a" * 40
|
||||
assert one[0]["x"]["delta.novelty"] == "new" and one[0]["x"]["delta.changed_lines"] == 1
|
||||
assert [r["x"] for r in one] == [r["x"] for r in two], "a commit id is envelope, never a feature"
|
||||
|
||||
|
||||
# --- label fields ------------------------------------------------------------------------------------------------
|
||||
|
||||
AGREED = {
|
||||
"family": "injection",
|
||||
"candidates": [
|
||||
{"candidate": "a.py::f", "site": "a.py:4:f", "a": True, "b": False, "c": True, "agreed": True, "site_match": True,
|
||||
"finding": {"family": "injection", "reported_at": "a.py:5"}},
|
||||
],
|
||||
"disputed": [],
|
||||
} # fmt: skip
|
||||
PASSES = {
|
||||
"a": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:5"}], "turns": 4}],
|
||||
"b": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [], "turns": 6}],
|
||||
"c": [{"path": "a.py", "candidates": [["a.py", "f", 4]], "flagged": [{"candidate": "a.py::f", "reported_at": "a.py:4"}], "turns": 5}],
|
||||
}
|
||||
ALERTS = [
|
||||
{"rule_id": "python-os-command", "rule_status": "enabled", "path": "a.py", "line": 5, "function": "f", "pin_role": "vulnerable",
|
||||
"in_fix_range": True},
|
||||
] # fmt: skip
|
||||
SUMMARY = {"status": "done", "quick_languages": ["python"], "engine_languages": ["php"]}
|
||||
|
||||
|
||||
def _plane(**overrides: Any) -> list[dict[str, Any]]:
|
||||
inputs: dict[str, Any] = {
|
||||
"facts": {"callers": {"a.py::f": [{"path": "b.py", "line": 2}]}}, "passes": PASSES, "models": {"a": "m", "b": "m", "c": "m"},
|
||||
"agreed": AGREED, "alerts": ALERTS, "alerts_summary": SUMMARY,
|
||||
} # fmt: skip
|
||||
return case_features(**{**inputs, **overrides})
|
||||
|
||||
|
||||
def test_label_fields_never_reach_features() -> None:
|
||||
baseline = _plane()
|
||||
assert [r["candidate"] for r in baseline] == ["a.py::f"] and baseline[0]["profile"] == "plane"
|
||||
x = baseline[0]["x"]
|
||||
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, False, True, 2)
|
||||
assert x["agree.final"] == "agreed" and x["qr.enabled_hits"] == 1 and x["verify.site_offset"] == 0
|
||||
flipped = copy.deepcopy(AGREED)
|
||||
flipped["candidates"][0]["site_match"] = False
|
||||
outside = [{**ALERTS[0], "in_fix_range": False}]
|
||||
fixed_pin = [*ALERTS, {**ALERTS[0], "pin_role": "fixed", "line": 9}, {**ALERTS[0], "pin_role": "fixed", "function": "g"}]
|
||||
for variant in (_plane(agreed=flipped), _plane(alerts=outside), _plane(alerts=fixed_pin)):
|
||||
assert variant == baseline, "site_match, in_fix_range and fixed-pin alerts change nothing"
|
||||
for field in ("site_match", "in_fix_range", "rule_id", "path", "repo", "usd"):
|
||||
smuggled = copy.deepcopy(baseline[0])
|
||||
smuggled["x"][field] = True
|
||||
with pytest.raises(FeatureRecordError, match="allow-list"):
|
||||
validate_record(smuggled)
|
||||
with pytest.raises(FeatureRecordError):
|
||||
validate_record({**baseline[0], field: True})
|
||||
|
||||
|
||||
# --- missing is not zero -----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_is_not_zero() -> None:
|
||||
uncovered = _plane(alerts=[], alerts_summary={**SUMMARY, "quick_languages": ["php"]})[0]
|
||||
assert uncovered["instruments"]["quick"]["state"] == "none"
|
||||
assert all(v is None for k, v in uncovered["x"].items() if k.startswith("qr."))
|
||||
covered = _plane(alerts=[])[0]
|
||||
assert covered["instruments"]["quick"]["state"] == "ran"
|
||||
assert covered["x"]["qr.enabled_hits"] == 0 and covered["x"]["qr.shadow_hits"] == 0, "ran and found nothing is 0"
|
||||
no_alerts = _plane(alerts=[], alerts_summary=None)[0]
|
||||
assert no_alerts["instruments"]["quick"]["state"] == "none" and no_alerts["x"]["qr.enabled_hits"] is None
|
||||
assert covered["instruments"]["engine"]["state"] == "none" and covered["x"]["eng.findings"] is None
|
||||
assert covered["instruments"]["delta"]["state"] == "not_applicable" and covered["x"]["delta.novelty"] is None
|
||||
failed = _plane(alerts_summary={**SUMMARY, "status": "failed"})[0]
|
||||
assert failed["instruments"]["quick"]["state"] == "failed" and failed["x"]["qr.enabled_hits"] is None
|
||||
php = {"path": "p.php", "line": 3, "function": "q", "pin_role": "vulnerable", "rule_id": "engine:injection", "rule_status": "enabled"}
|
||||
engine = {**SUMMARY, "engine": {"vulnerable": {"questions": 0, "completed": 0, "files": 5, "degradations": []}}}
|
||||
[_, unread] = _plane(alerts=[*ALERTS, php], alerts_summary=engine)
|
||||
assert unread["instruments"]["engine"]["state"] == "failed" and unread["x"]["eng.findings"] is None, "no question is not zero"
|
||||
|
||||
|
||||
def test_build_for_scan_marks_an_engine_that_did_not_run_or_read(tmp_path: Path) -> None:
|
||||
root = _repo(tmp_path / "r", "app/one.py", "alpha", "beta")
|
||||
_, targets = preprocess_repository(root)
|
||||
findings = quick_scan_findings(root, targets, rank_targets(targets))
|
||||
unread = build_for_scan(root, findings, {"questions": 0, "findings": []})
|
||||
absent = build_for_scan(root, findings, None)
|
||||
assert {r["instruments"]["engine"]["state"] for r in unread} == {"failed"}
|
||||
assert {r["instruments"]["engine"]["state"] for r in absent} == {"none"}
|
||||
assert all(r["x"]["eng.findings"] is None for r in (*unread, *absent))
|
||||
|
||||
|
||||
def test_validate_record_refuses_values_outside_their_type() -> None:
|
||||
good = _plane()[0]
|
||||
validate_record(good)
|
||||
for name, value in (("agree.final", "maybe"), ("verify.votes", 4), ("qr.enabled_hits", -1), ("lang.python", 1)):
|
||||
bad = copy.deepcopy(good)
|
||||
bad["x"][name] = value
|
||||
with pytest.raises(FeatureRecordError, match=name.replace(".", r"\.")):
|
||||
validate_record(bad)
|
||||
null = copy.deepcopy(good)
|
||||
null["x"]["agree.final"] = None
|
||||
with pytest.raises(FeatureRecordError, match="null is not allowed"):
|
||||
validate_record(null)
|
||||
value = copy.deepcopy(good)
|
||||
value["x"]["ms.flagged"] = False
|
||||
with pytest.raises(FeatureRecordError, match="must be null"):
|
||||
validate_record(value)
|
||||
with pytest.raises(FeatureRecordError, match="outside the static profile"):
|
||||
record("a.py::f", "injection", "python", "static", {"verify": Part("ran", {})})
|
||||
@@ -31,7 +31,7 @@ def _benchmark(name: str, *, reported_only: bool): # type: ignore[no-untyped-de
|
||||
|
||||
def test_php_rules_load_compile_and_are_php_scoped() -> None:
|
||||
rules = _php_rules()
|
||||
assert len(rules) == 13
|
||||
assert len(rules) == 14, "13 measured rules; php-unserialize split into its language and WordPress parts (task 2)"
|
||||
for rule in rules.values():
|
||||
assert rule.languages == ("php",) and rule.status in {"enabled", "shadow"}
|
||||
re.compile(rule.pattern)
|
||||
@@ -52,6 +52,7 @@ def test_every_php_rule_sink_is_in_the_engines_vocabulary() -> None:
|
||||
def test_status_and_precision_are_the_committed_measurement() -> None:
|
||||
measured = json.loads(MEASUREMENT.read_text())["rules"]
|
||||
for rule_id, rule in _php_rules().items():
|
||||
rule_id = "php-unserialize" if rule_id == "php-maybe-unserialize" else rule_id # split from it, keeps its numbers
|
||||
assert rule.status == measured[rule_id]["status"], rule_id
|
||||
assert rule.precision_estimate == measured[rule_id]["precision_lower_bound"], rule_id
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""The plane task `features` (learned-decision-engine task 1) on validation-46's recorded lmdeploy case
|
||||
(``tests/fixtures/plane-remember``), and `remember` storing its records as `features` rows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.learn.schema import FeatureRecordError, feature_set_digest
|
||||
from openultrasast.plane.memory import FileStore
|
||||
from openultrasast.plane.tasks import features
|
||||
from openultrasast.plane.tasks.remember import Context, remember_run, rows_for
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "plane-remember" / "validation-46"
|
||||
CASE = "lmdeploy-media-url-ssrf"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "plane" / "validation-46"
|
||||
shutil.copytree(FIXTURE, target)
|
||||
return target
|
||||
|
||||
|
||||
def _checkout(root: Path) -> Path:
|
||||
media = root / "lmdeploy" / "vl" / "media"
|
||||
media.mkdir(parents=True)
|
||||
(media / "connection.py").write_text("def _is_safe_url(u):\n return True\n\n\ndef _load_http_url(u):\n x = 1\n return x\n")
|
||||
return root
|
||||
|
||||
|
||||
def test_features_of_the_recorded_case(run_dir: Path, tmp_path: Path) -> None:
|
||||
[rec] = features.recorded_case(run_dir, CASE, workspace=_checkout(tmp_path / "checkout"))
|
||||
assert (rec["candidate"], rec["family"], rec["profile"], rec["unit"]) == (
|
||||
"lmdeploy/vl/media/connection.py::_load_http_url", "untrusted_destination", "plane", "pin",
|
||||
) # fmt: skip
|
||||
x, states = rec["x"], {k: v["state"] for k, v in rec["instruments"].items()}
|
||||
assert (x["verify.flag_a"], x["verify.flag_b"], x["verify.flag_c"], x["verify.votes"]) == (True, True, None, 2)
|
||||
assert x["agree.final"] == "agreed" and x["facts.callers"] == 0 and x["facts.is_global"] is False
|
||||
assert x["facts.function_lines"] == 3 and x["lang.python"] is True and x["verify.site_offset"] == 0
|
||||
assert states["verify"] == states["agree"] == states["facts"] == states["source"] == "ran"
|
||||
assert states["quick"] == states["engine"] == "none", "no alerts task in this Run: no coverage, not zero hits"
|
||||
assert rec["instruments"]["verify"]["version"] == "deepseek-flash" and rec["feature_set_digest"] == feature_set_digest()
|
||||
assert "site_match" not in json.dumps(rec), "the fixture's site_match: true was dropped on read"
|
||||
[without] = features.recorded_case(run_dir, CASE)
|
||||
assert without["instruments"]["source"]["state"] == "none" and without["x"]["facts.function_lines"] is None
|
||||
|
||||
|
||||
def test_the_task_writes_records_and_remember_stores_them(run_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_INPUT_FACTS": str(run_dir / f"{CASE}-facts" / "facts.json"),
|
||||
"OUSAST_INPUT_AGREED": str(run_dir / f"{CASE}-final" / "agreed.json"),
|
||||
**{f"OUSAST_INPUT_PASS_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "units.jsonl") for p in "abc"},
|
||||
**{f"OUSAST_INPUT_SUMMARY_{p.upper()}": str(run_dir / f"{CASE}-v{p}" / "summary.json") for p in "abc"},
|
||||
} # fmt: skip
|
||||
assert features.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert (summary["status"], summary["records"], summary["usd"], summary["calls"]) == ("done", 1, 0, 0)
|
||||
assert summary["instruments"]["verify"] == {"ran": 1} and summary["instruments"]["quick"] == {"none": 1}
|
||||
(run_dir / f"{CASE}-features").mkdir()
|
||||
shutil.copy(out / features.OUTPUT, run_dir / f"{CASE}-features" / features.OUTPUT)
|
||||
store = FileStore(tmp_path / "memory")
|
||||
[result] = remember_run(run_dir, store, population="population-v2", split="validation")
|
||||
assert result.kinds == {"facts": 1, "unit_cost": 2, "verdict": 1, "features": 1}
|
||||
[row] = store.rows(kind="features")
|
||||
assert row.row["candidate"] == "lmdeploy/vl/media/connection.py::_load_http_url" and row.row["family"] == "untrusted_destination"
|
||||
assert row.row["repo"] == "example.com/reserved/lmdeploy" and row.row["x"]["agree.final"] == "agreed"
|
||||
|
||||
|
||||
def test_remember_refuses_a_record_outside_the_allow_list(run_dir: Path) -> None:
|
||||
[rec] = features.recorded_case(run_dir, CASE)
|
||||
rec["x"]["site_match"] = True
|
||||
ctx = Context("r", "c-remember", "https://github.com/o/r.git", "0" * 40, "sha256:" + "1" * 64)
|
||||
with pytest.raises(FeatureRecordError, match="site_match"):
|
||||
rows_for(ctx, facts=None, passes={}, models={}, agreed=None, features=[rec])
|
||||
|
||||
|
||||
def test_the_task_fails_loudly_without_an_output_dir() -> None:
|
||||
assert features.main({}) == 2
|
||||
@@ -56,7 +56,8 @@ def test_model_prices_are_the_endpoint_prices() -> None:
|
||||
|
||||
def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
|
||||
run, manifests = reconciler.load_run(RUN)
|
||||
assert len(run.tasks) == 105 # per case: facts, va, vb, agree, then vc, final, then remember (harnessx-removal task 4)
|
||||
# per case: facts, va, vb, agree, then vc, final, then features (learned-decision-engine task 1) and remember
|
||||
assert len(run.tasks) == 120
|
||||
population = {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]}
|
||||
ax_names: set[str] = set()
|
||||
for task in manifests.tasks.values():
|
||||
@@ -96,7 +97,9 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical(
|
||||
first = text.split(" - name: budibase-mongo-template-nosqli-vc\n", 1)[0]
|
||||
assert hashlib.sha256(first.encode()).hexdigest() == FIRST_RUN_ENTRIES
|
||||
run, manifests = reconciler.load_run(RUN)
|
||||
old, new, remembered = run.tasks[:60], run.tasks[60:90], run.tasks[90:]
|
||||
old, new, appended = run.tasks[:60], run.tasks[60:90], run.tasks[90:]
|
||||
assert [e.name.rsplit("-", 1)[1] for e in appended] == ["features", "remember"] * 15
|
||||
remembered = appended[1::2]
|
||||
assert [e.name.rsplit("-", 1)[1] for e in old] == ["facts", "va", "vb", "agree"] * 15
|
||||
assert [e.name.rsplit("-", 1)[1] for e in new] == ["vc", "final"] * 15
|
||||
for vc, final in zip(new[::2], new[1::2], strict=True):
|
||||
@@ -112,9 +115,15 @@ def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical(
|
||||
for entry in (vc, final):
|
||||
name = reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)
|
||||
assert len(name) <= reconciler.TASK_NAME_LIMIT
|
||||
for entry in appended[::2]: # learned-decision-engine task 1: model-free, the chain's artifacts in, records out
|
||||
case = entry.name.removesuffix("-features")
|
||||
assert entry.inputs["agreed"] == f"{case}-final/agreed.json" and entry.outputs == ("features.jsonl", "summary.json")
|
||||
assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0)
|
||||
assert reconciler.MODEL_ANNOTATION not in manifests.tasks[entry.task].metadata.annotations
|
||||
for entry in remembered: # harnessx-removal task 4: appended after the tie-break, model-free
|
||||
case = entry.name.removesuffix("-remember")
|
||||
assert entry.inputs["agreed"] == f"{case}-final/agreed.json" and entry.inputs["pass_c"] == f"{case}-vc/units.jsonl"
|
||||
assert entry.inputs["features"] == f"{case}-features/features.jsonl"
|
||||
assert entry.budget is not None and (entry.budget.usd, entry.budget.calls) == (0, 0)
|
||||
assert reconciler.MODEL_ANNOTATION not in manifests.tasks[entry.task].metadata.annotations
|
||||
assert len(reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)) <= reconciler.TASK_NAME_LIMIT
|
||||
|
||||
@@ -169,15 +169,18 @@ def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
|
||||
def test_every_case_ends_in_remember_and_the_loop_is_model_free(tmp_path: Path) -> None:
|
||||
plain = render([_case()], _templates(), "validation-46", "t", population="population-v2", split="validation")
|
||||
run, _ = reconciler.load_run(_write(tmp_path / "plain", plain))
|
||||
assert [e.name.removeprefix(f"{CASE}-") for e in run.tasks] == [*CHAIN[:4], "vc", "final", "remember"]
|
||||
assert [e.name.removeprefix(f"{CASE}-") for e in run.tasks] == [*CHAIN[:4], "vc", "final", "features", "remember"]
|
||||
remember = run.task(f"{CASE}-remember")
|
||||
assert remember.inputs["agreed"] == f"{CASE}-final/agreed.json" and "alerts" not in remember.inputs
|
||||
assert set(remember.producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final")}
|
||||
assert set(remember.producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final", "features")}
|
||||
assert set(run.task(f"{CASE}-features").producers) == {f"{CASE}-{step}" for step in ("facts", "va", "vb", "vc", "final")}
|
||||
|
||||
files = render([_case(fixed_ranges={"x.py": [(1, 2)]})], _templates(), "validation-46", "t", population="p", split="s", loop=LOOP)
|
||||
run, manifests = reconciler.load_run(_write(tmp_path / "loop", files))
|
||||
names = [e.name for e in run.tasks]
|
||||
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN] and names[6:] == [f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS]
|
||||
assert names[:6] == [f"{CASE}-{step}" for step in CHAIN]
|
||||
assert names[6:] == [f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", *LOOP_TASKS]
|
||||
assert run.task(f"{CASE}-features").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl", "features read the alerts"
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts"] == f"{CASE}-alerts/alerts.jsonl"
|
||||
assert run.task(f"{CASE}-remember").inputs["alerts_summary"] == f"{CASE}-alerts/summary.json", "the coverage travels too"
|
||||
measure_inputs = {f"remember_{CASE}": f"{CASE}-remember/memory.jsonl", "snapshot": "memory-snapshot/rows.jsonl"}
|
||||
@@ -262,6 +265,7 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
fake.script({
|
||||
f"{CASE}-alerts": {"exec": {"env": {"PYTHONPATH": SRC, "OUSAST_WORKSPACE_DIR": str(vulnerable), "OUSAST_FIXED_DIR": str(fixed),
|
||||
"OUSAST_INPUT_CASE": str(tmp_path / "case.json")}}},
|
||||
f"{CASE}-features": {"exec": {"env": {"PYTHONPATH": SRC, "OUSAST_WORKSPACE_DIR": str(vulnerable)}}},
|
||||
f"{CASE}-remember": {"exec": {"env": {"PYTHONPATH": SRC}}},
|
||||
"loop-measure": {"exec": {"env": {"PYTHONPATH": SRC}}},
|
||||
"loop-propose": {"exec": {"env": project_env}},
|
||||
@@ -271,10 +275,10 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
assert cli.main(["plane", "run", str(manifest), "--ax", str(fake.ax)]) == 0, (fake.root / "ax.log").read_text()[-3000:]
|
||||
|
||||
statuses = {name: info["status"] for name, info in json.loads((base / "state.json").read_text())["tasks"].items()}
|
||||
assert set(statuses.values()) == {"done"} and len(statuses) == 12
|
||||
assert set(statuses.values()) == {"done"} and len(statuses) == 13
|
||||
assert json.loads((base / "memory-snapshot" / "index.json").read_text())["kept"] == 4, "seeded on the host, not run"
|
||||
delivered = [name for _, name in fake.events("delivered")]
|
||||
assert delivered == [f"{CASE}-alerts", f"{CASE}-remember", "loop-measure", "loop-propose", "loop-improve"]
|
||||
assert delivered == [f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", "loop-measure", "loop-propose", "loop-improve"]
|
||||
|
||||
measure = json.loads((base / "loop-measure" / "measure.json").read_text())
|
||||
assert measure["run_metrics"]["declared_sites_agreed"] == 1 and measure["run_metrics"]["candidates"] == 1
|
||||
@@ -297,10 +301,13 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos
|
||||
|
||||
report = reconciler.attribution("validation-46")
|
||||
for row in report["tasks"]:
|
||||
if row["task"] in (f"{CASE}-alerts", f"{CASE}-remember", *LOOP_TASKS):
|
||||
if row["task"] in (f"{CASE}-alerts", f"{CASE}-features", f"{CASE}-remember", *LOOP_TASKS):
|
||||
assert row["model"] is None and row["usd"] in (0, None) and row["calls"] in (0, None), row
|
||||
ingested = FileStore(store.root)
|
||||
outcomes = ingested.rows(kind="proposal_outcome")
|
||||
assert {r.row["outcome"] for r in outcomes} == {"reverted"} and len(outcomes) == 2, "one per (repo, pin) of the evidence"
|
||||
[fixed_alert] = ingested.rows(repo="example.com/reserved/lmdeploy", pin=FIXED, kind="alert")
|
||||
features = {r.row["candidate"]: r.row for r in ingested.rows(kind="features")}
|
||||
assert set(features) == {"lmdeploy/vl/media/connection.py::_load_http_url"}, "no fixed-pin alert became a candidate"
|
||||
assert all(r["pin"] == VULNERABLE and "in_fix_range" not in json.dumps(r["x"]) for r in features.values())
|
||||
assert fixed_alert.row["in_fix_range"] is True and fixed_alert.row["function"] == "load"
|
||||
|
||||
@@ -19,7 +19,7 @@ from openultrasast.ruleset import (
|
||||
def test_default_ruleset_loads_as_data() -> None:
|
||||
rules = load_ruleset(DEFAULT_RULESET_DIR)
|
||||
php = [rule for rule in rules if rule.languages == ("php",)]
|
||||
assert len(rules) == 32 + len(php) and len(php) == 13
|
||||
assert len(rules) == 32 + len(php) and len(php) == 14 # php-unserialize split in two (decision-engine task 2)
|
||||
assert all(rule.status == "enabled" for rule in rules if rule not in php), "PHP rules carry their measured status"
|
||||
assert all(not hasattr(rule, "severity") for rule in rules) # severity is policy-governed, not rule-local
|
||||
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Framework knowledge as tagged priors (learned-decision-engine task 2; Req 8.2, design section 2).
|
||||
|
||||
The lint: no untagged entry of ``ruleset/semantic/*.toml`` or of a quick ``rules.toml`` contains a symbol of
|
||||
``ruleset/frameworks.toml`` (a symbol of a framework or library of the entry's language, matched as a whole token;
|
||||
a quick rule's pattern is read with its regex escapes removed). ``priors="off"`` removes exactly the tagged entries
|
||||
and activates the language-level fallbacks of mixed quick rules; the default ``"all"`` is today's ruleset.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.ruleset import DEFAULT_RULESET_DIR, load_ruleset
|
||||
from openultrasast.ruleset.frameworks import FrameworksError, framework_ids, load_frameworks, symbol_in
|
||||
from openultrasast.semantic.facts import DEFAULT_FACTS_DIR, load_facts
|
||||
|
||||
TABLES = ("source", "sink", "sanitizer", "dispatch")
|
||||
LANGUAGE_FAMILY = {"typescript": "javascript", "cpp": "c"}
|
||||
|
||||
|
||||
def _lexicon(language: str) -> dict[str, tuple[str, ...]]:
|
||||
wanted = LANGUAGE_FAMILY.get(language, language)
|
||||
return {row.id: row.symbols for row in load_frameworks() if wanted in {LANGUAGE_FAMILY.get(lang, lang) for lang in row.languages}}
|
||||
|
||||
|
||||
def _texts(item: dict[str, object]) -> list[str]:
|
||||
keys = ("patterns", "calls", "register", "apply", "returns", "request_arguments", "origin_anchors")
|
||||
return [str(text) for key in keys for text in item.get(key, []) or []] # type: ignore[union-attr]
|
||||
|
||||
|
||||
def _semantic_entries() -> list[tuple[str, str, dict[str, object]]]:
|
||||
rows = []
|
||||
for path in sorted(DEFAULT_FACTS_DIR.glob("*.toml")):
|
||||
data = tomllib.loads(path.read_text())
|
||||
rows += [
|
||||
(str(data.get("language") or path.stem), f"{path.name} {table} {item['id']}", item)
|
||||
for table in TABLES
|
||||
for item in data.get(table, [])
|
||||
]
|
||||
return rows
|
||||
|
||||
|
||||
def _rules() -> list[dict[str, object]]:
|
||||
return [rule for path in sorted(DEFAULT_RULESET_DIR.glob("*/rules.toml")) for rule in tomllib.loads(path.read_text()).get("rule", [])]
|
||||
|
||||
|
||||
def _unescaped(pattern: str) -> str:
|
||||
return re.sub(r"\\(.)", r"\1", pattern)
|
||||
|
||||
|
||||
def test_no_untagged_entry_names_a_framework_symbol() -> None:
|
||||
offenders: list[str] = []
|
||||
for language, where, item in _semantic_entries():
|
||||
if item.get("framework") or item.get("library"):
|
||||
continue
|
||||
for framework, symbols in _lexicon(language).items():
|
||||
hits = sorted({s for s in symbols for text in _texts(item) if symbol_in(s, text)})
|
||||
if hits:
|
||||
offenders.append(f"{where}: {framework} {hits}")
|
||||
for rule in _rules():
|
||||
if rule.get("framework") or rule.get("library"):
|
||||
continue
|
||||
text = _unescaped(str(rule["pattern"]))
|
||||
for language in rule.get("languages") or []: # type: ignore[union-attr]
|
||||
for framework, symbols in _lexicon(str(language)).items():
|
||||
hits = sorted(s for s in symbols if symbol_in(s, text))
|
||||
if hits:
|
||||
offenders.append(f"rule {rule['rule_id']}: {framework} {hits}")
|
||||
assert not offenders, "untagged framework knowledge (tag it, or split the entry):\n" + "\n".join(sorted(set(offenders)))
|
||||
|
||||
|
||||
def test_the_lint_catches_an_untagged_framework_entry() -> None:
|
||||
item = {"id": "x", "calls": ["$wpdb->get_var", "mysqli_query"]}
|
||||
assert any(symbol_in(s, t) for s in _lexicon("php")["wordpress"] for t in _texts(item))
|
||||
assert not any(symbol_in(s, "maybe_unserialized_value") for s in ("maybe_unserialize",)), "whole tokens only"
|
||||
assert symbol_in("wpdb->", _unescaped(r"\$(?!wpdb->)[A-Za-z_]")), "a symbol inside an exclusion still counts"
|
||||
|
||||
|
||||
def test_every_tag_names_a_listed_id_and_the_lexicon_is_well_formed() -> None:
|
||||
ids = framework_ids()
|
||||
tags = [item.get("framework") or item.get("library") for _, _, item in _semantic_entries()]
|
||||
tags += [rule.get("framework") or rule.get("library") for rule in _rules()]
|
||||
assert {t for t in tags if t} <= ids
|
||||
assert {"wordpress", "flask", "django", "express", "spring", "jakarta-servlet"} <= ids
|
||||
for row in load_frameworks():
|
||||
assert row.symbols and row.languages and row.kind in ("framework", "library"), row.id
|
||||
|
||||
|
||||
def _signature(facts: object) -> set[tuple[str, str, str, tuple[str, ...]]]:
|
||||
rows = set()
|
||||
for kind in ("sources", "sinks", "sanitizers", "dispatches"):
|
||||
for fact in getattr(facts, kind):
|
||||
texts = tuple(getattr(fact, "patterns", ()) or getattr(fact, "calls", ()) or getattr(fact, "register", ()))
|
||||
rows.add((kind, fact.language, fact.id, texts))
|
||||
return rows
|
||||
|
||||
|
||||
def test_priors_off_removes_exactly_the_tagged_entries() -> None:
|
||||
every, off = load_facts(), load_facts(priors="off")
|
||||
assert load_facts(priors="all") is every, "the default is today's facts, memoised"
|
||||
tagged = {
|
||||
row for row in _signature(every)
|
||||
for fact in getattr(every, row[0]) if (fact.language, fact.id) == row[1:3]
|
||||
and tuple(getattr(fact, "patterns", ()) or getattr(fact, "calls", ()) or getattr(fact, "register", ())) == row[3]
|
||||
and (fact.framework or fact.library)
|
||||
} # fmt: skip
|
||||
assert tagged, "the semantic facts carry tagged priors"
|
||||
assert _signature(every) - _signature(off) == tagged and _signature(off) <= _signature(every)
|
||||
assert off.layouts == every.layouts, "layouts are product filters, not detection"
|
||||
rules_all = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR)}
|
||||
rules_off = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR, priors="off")}
|
||||
tagged_rules = {rid for rid, r in rules_all.items() if r.framework or r.library}
|
||||
fallbacks = {rid for rid, r in rules_off.items() if r.fallback_for}
|
||||
assert tagged_rules and set(rules_all) - set(rules_off) == tagged_rules
|
||||
assert set(rules_off) - set(rules_all) == fallbacks and {rules_off[f].fallback_for for f in fallbacks} <= tagged_rules
|
||||
assert not any(r.fallback_for for r in rules_all.values()), "a fallback is inactive while its prior is on"
|
||||
|
||||
|
||||
def test_split_entries_keep_their_language_level_calls() -> None:
|
||||
off = load_facts(priors="off")
|
||||
|
||||
def calls(language: str, kind: str, ident: str) -> set[str]:
|
||||
scoped = off.for_language(language)
|
||||
return {c for f in getattr(scoped, kind) if f.id == ident for c in (getattr(f, "calls", ()) or getattr(f, "patterns", ()))}
|
||||
|
||||
assert calls("php", "sinks", "unserialize") == {"unserialize"}
|
||||
assert calls("php", "sanitizers", "sql_escape") == {"mysqli_real_escape_string", "mysql_real_escape_string", "pg_escape_string"}
|
||||
assert calls("php", "sanitizers", "coerce") == {"intval", "floatval", "filter_var", "settype"}
|
||||
assert calls("php", "sinks", "wpdb") == set() and not off.for_language("php").dispatches
|
||||
assert calls("python", "sinks", "pickle.loads") == {"pickle.loads", "pickle.load", "marshal.loads", "cPickle.loads"}
|
||||
assert calls("python", "sinks", "path.open") == {"open", "os.remove", "os.unlink", "shutil.copy", "shutil.move", "shutil.rmtree"}
|
||||
assert calls("python", "sinks", "outbound.request") == {"urlopen", "urlretrieve"} and calls("python", "sources", "request") == set()
|
||||
assert calls("javascript", "sinks", "outbound.request") == {"fetch", "http.get", "http.request", "https.get", "https.request"}
|
||||
assert calls("javascript", "sinks", "prototype.merge") == {"Object.assign"}
|
||||
assert calls("javascript", "sanitizers", "html.escape") == {"encodeURIComponent", "encodeURI", "escape"}
|
||||
assert calls("java", "sinks", "outbound.request") == {"openConnection", "openStream"}
|
||||
assert calls("java", "sanitizers", "java.escape") == {"getCanonicalPath", "normalize", "URLEncoder.encode"}
|
||||
assert calls("java", "sources", "servlet") == {"readLine"}
|
||||
rules = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR, priors="off")}
|
||||
assert "php-unserialize" in rules and "php-maybe-unserialize" not in rules
|
||||
assert re.search(rules["php-unserialize"].pattern, "$x = unserialize($_POST['a']);")
|
||||
assert not re.search(rules["php-unserialize"].pattern, "$x = maybe_unserialize($_POST['a']);")
|
||||
assert re.search(rules["python-unsafe-deserialization-language"].pattern, "pickle.loads(data)")
|
||||
assert not re.search(rules["python-unsafe-deserialization-language"].pattern, "yaml.load(data)")
|
||||
|
||||
|
||||
def test_a_chosen_prior_keeps_only_its_own_entries() -> None:
|
||||
wordpress = load_facts(priors=frozenset({"wordpress"}))
|
||||
assert {c for f in wordpress.for_language("php").sinks if f.id == "wpdb" for c in f.calls} >= {"$wpdb->get_var"}
|
||||
assert not any(f.framework == "flask" for f in wordpress.sources)
|
||||
rules = {r.rule_id for r in load_ruleset(DEFAULT_RULESET_DIR, priors=frozenset({"wordpress"}))}
|
||||
assert "php-wpdb-sql-composition" in rules and "python-ssti" not in rules and "python-unsafe-deserialization-language" in rules
|
||||
with pytest.raises(FrameworksError, match="unknown priors"):
|
||||
load_facts(priors=frozenset({"no-such-framework"}))
|
||||
with pytest.raises(FrameworksError):
|
||||
load_ruleset(DEFAULT_RULESET_DIR, priors="none") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_tag_outside_frameworks_toml_is_refused(tmp_path: Path) -> None:
|
||||
(tmp_path / "x.toml").write_text('language = "php"\n[[sink]]\nid = "s"\nframework = "nope"\ncwe = "CWE-89"\ncalls = ["q"]\n')
|
||||
with pytest.raises(ValueError, match="not a framework id"):
|
||||
load_facts(tmp_path)
|
||||
Reference in New Issue
Block a user