From fe300e635223b0377c5416aad12a8b0db2e402b5 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:49:44 +0200 Subject: [PATCH] feat(three-stage-scan): add split-sink fixtures the regex scan misses --- .kiro/specs/three-stage-scan/tasks.md | 4 +- benchmarks/fixtures/split-sink-c/src/vuln.c | 8 +++ .../main/java/example/SplitSinkServlet.java | 12 ++++ .../fixtures/split-sink-javascript/src/app.js | 12 ++++ benchmarks/fixtures/split-sink-python/app.py | 11 +++ benchmarks/manifests/split-sink-c.toml | 20 ++++++ benchmarks/manifests/split-sink-java.toml | 20 ++++++ .../manifests/split-sink-javascript.toml | 19 +++++ benchmarks/manifests/split-sink-python.toml | 20 ++++++ tests/test_benchmark.py | 4 ++ tests/test_split_sink_fixtures.py | 70 +++++++++++++++++++ 11 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 benchmarks/fixtures/split-sink-c/src/vuln.c create mode 100644 benchmarks/fixtures/split-sink-java/src/main/java/example/SplitSinkServlet.java create mode 100644 benchmarks/fixtures/split-sink-javascript/src/app.js create mode 100644 benchmarks/fixtures/split-sink-python/app.py create mode 100644 benchmarks/manifests/split-sink-c.toml create mode 100644 benchmarks/manifests/split-sink-java.toml create mode 100644 benchmarks/manifests/split-sink-javascript.toml create mode 100644 benchmarks/manifests/split-sink-python.toml create mode 100644 tests/test_split_sink_fixtures.py diff --git a/.kiro/specs/three-stage-scan/tasks.md b/.kiro/specs/three-stage-scan/tasks.md index 436954a..b7d45f5 100644 --- a/.kiro/specs/three-stage-scan/tasks.md +++ b/.kiro/specs/three-stage-scan/tasks.md @@ -156,7 +156,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex - _Requirements: 6.1, 6.3, 6.5, 6.6_ - _Depends: 3.2, 6.2_ -- [ ] 6.5 Surface map, hints, and worth-fixing in the report and honor fail-on +- [x] 6.5 Surface map, hints, and worth-fixing in the report and honor fail-on - Markdown, manifest, and SARIF remain; the report gains map and worth-fixing sections. - `--fail-on worth-fixing` fails only when a worth-fixing verdict exists. - Observable completion: a scan with one worth-fixing verdict fails under that flag and passes under `--fail-on never`. @@ -166,7 +166,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex ## Phase 5 — Validation: Grounded corpora and CI - [ ] 7. Replace cheat-sheet success for stages 2 and 3 -- [ ] 7.1 Add split-sink fixtures for Python, JavaScript, Java, and C/C++ +- [x] 7.1 Add split-sink fixtures for Python, JavaScript, Java, and C/C++ - Each language family has at least one case where source and sink are on different lines or assigned before use. - Observable completion: stage-1 pattern scan misses at least one planted sink in each family, documented in the manifest expected set. - _Requirements: 8.1_ diff --git a/benchmarks/fixtures/split-sink-c/src/vuln.c b/benchmarks/fixtures/split-sink-c/src/vuln.c new file mode 100644 index 0000000..5515e64 --- /dev/null +++ b/benchmarks/fixtures/split-sink-c/src/vuln.c @@ -0,0 +1,8 @@ +#include +#include + +void search(sqlite3 *db, char *term) { + char query[512]; + snprintf(query, sizeof(query), "SELECT * FROM items WHERE title LIKE '%%%s%%'", term); + sqlite3_exec(db, query, 0, 0, 0); /* CWE-89 split-sink: query built on prior line (regex blind spot) */ +} diff --git a/benchmarks/fixtures/split-sink-java/src/main/java/example/SplitSinkServlet.java b/benchmarks/fixtures/split-sink-java/src/main/java/example/SplitSinkServlet.java new file mode 100644 index 0000000..c9503df --- /dev/null +++ b/benchmarks/fixtures/split-sink-java/src/main/java/example/SplitSinkServlet.java @@ -0,0 +1,12 @@ +package example; + +import jakarta.servlet.http.HttpServletRequest; +import java.sql.Connection; + +class SplitSinkServlet { + void search(HttpServletRequest request, Connection connection) throws Exception { + String term = request.getParameter("q"); + String sql = String.format("select * from items where title like '%%%s%%'", term); + connection.createStatement().executeQuery(sql); // CWE-89 split-sink: sql built on prior line (regex blind spot) + } +} diff --git a/benchmarks/fixtures/split-sink-javascript/src/app.js b/benchmarks/fixtures/split-sink-javascript/src/app.js new file mode 100644 index 0000000..7fcfade --- /dev/null +++ b/benchmarks/fixtures/split-sink-javascript/src/app.js @@ -0,0 +1,12 @@ +// Split-sink SQL injection: query is built, then executed on the next line. +const express = require("express"); +const app = express(); + +app.get("/search", (req, res) => { + const term = req.query.q; + const query = "select * from items where title like '%" + term + "%'"; + db.query(query); // CWE-89 split-sink: query built on prior line (regex blind spot) + res.send("ok"); +}); + +module.exports = app; diff --git a/benchmarks/fixtures/split-sink-python/app.py b/benchmarks/fixtures/split-sink-python/app.py new file mode 100644 index 0000000..d14a386 --- /dev/null +++ b/benchmarks/fixtures/split-sink-python/app.py @@ -0,0 +1,11 @@ +"""Split-sink SQL injection: query is built, then executed on the next line.""" +from flask import Flask, request + +app = Flask(__name__) + + +@app.get("/search") +def search(db): + term = request.args["q"] + query = "select * from items where title like '%" + term + "%'" + return db.execute(query) # CWE-89 split-sink: query built on prior line (regex blind spot) diff --git a/benchmarks/manifests/split-sink-c.toml b/benchmarks/manifests/split-sink-c.toml new file mode 100644 index 0000000..38ad095 --- /dev/null +++ b/benchmarks/manifests/split-sink-c.toml @@ -0,0 +1,20 @@ +name = "split-sink-c" +language = "c_cpp" +frameworks = ["native"] +setup = [] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/split-sink-c" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/vuln.c" +line = 7 +function = "search" +sink = "sqlite3_exec" +evidence = "sqlite3_exec runs a SQL query string built from attacker input on a prior line" diff --git a/benchmarks/manifests/split-sink-java.toml b/benchmarks/manifests/split-sink-java.toml new file mode 100644 index 0000000..133f28b --- /dev/null +++ b/benchmarks/manifests/split-sink-java.toml @@ -0,0 +1,20 @@ +name = "split-sink-java" +language = "java_web" +frameworks = ["servlet"] +setup = ["mvn test"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/split-sink-java" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/main/java/example/SplitSinkServlet.java" +line = 10 +function = "search" +sink = "executeQuery" +evidence = "executeQuery runs a SQL string built from request input on a prior line" diff --git a/benchmarks/manifests/split-sink-javascript.toml b/benchmarks/manifests/split-sink-javascript.toml new file mode 100644 index 0000000..14881f3 --- /dev/null +++ b/benchmarks/manifests/split-sink-javascript.toml @@ -0,0 +1,19 @@ +name = "split-sink-javascript" +language = "javascript_node_web" +frameworks = ["node", "express"] +setup = ["npm install"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/split-sink-javascript" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/app.js" +line = 8 +sink = "query" +evidence = "query runs a SQL string built from request input on a prior line" diff --git a/benchmarks/manifests/split-sink-python.toml b/benchmarks/manifests/split-sink-python.toml new file mode 100644 index 0000000..55329a5 --- /dev/null +++ b/benchmarks/manifests/split-sink-python.toml @@ -0,0 +1,20 @@ +name = "split-sink-python" +language = "python_web" +frameworks = ["flask"] +setup = ["pip install flask"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/split-sink-python" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "app.py" +line = 11 +function = "search" +sink = "execute" +evidence = "execute runs a SQL query string built from request input on a prior line" diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index bc4cd99..52b500d 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -65,6 +65,10 @@ def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None: "javascript-vulnerable.toml", "python-vulnerable.toml", "python-web-smoke.toml", + "split-sink-c.toml", + "split-sink-java.toml", + "split-sink-javascript.toml", + "split-sink-python.toml", } for manifest_path in manifest_paths: manifest = load_benchmark_manifest(manifest_path) diff --git a/tests/test_split_sink_fixtures.py b/tests/test_split_sink_fixtures.py new file mode 100644 index 0000000..e49a073 --- /dev/null +++ b/tests/test_split_sink_fixtures.py @@ -0,0 +1,70 @@ +"""Split-sink fixtures: stage-1 pattern scan must miss planted sinks (requirement 8.1).""" + +from pathlib import Path + +import pytest + +from openultrasast.benchmark import ( + BenchmarkManifest, + BenchmarkRun, + ExpectedFinding, + evaluate_benchmark, + load_benchmark_manifest, + resolve_benchmark_source, +) +from openultrasast.findings import StaticFinding, quick_scan_findings +from openultrasast.gate import LANGUAGE_MANIFESTS +from openultrasast.preprocess import preprocess_repository +from openultrasast.rank import rank_targets + +MANIFEST_DIR = Path("benchmarks/manifests") +SPLIT_SINK_MANIFESTS = ( + "split-sink-c", + "split-sink-java", + "split-sink-javascript", + "split-sink-python", +) + + +def _quick_scan(target: Path) -> list[StaticFinding]: + _, targets = preprocess_repository(target) + return quick_scan_findings(target, targets, rank_targets(targets)) + + +def _planted_split_sinks(manifest: BenchmarkManifest) -> list[ExpectedFinding]: + return [item for item in manifest.expected if item.rule_id is None] + + +@pytest.mark.parametrize("name", SPLIT_SINK_MANIFESTS) +def test_stage1_misses_planted_split_sink(name: str) -> None: + manifest_path = MANIFEST_DIR / f"{name}.toml" + manifest = load_benchmark_manifest(manifest_path) + target = resolve_benchmark_source(manifest_path, manifest) + planted = _planted_split_sinks(manifest) + assert planted, f"{name} must document a stage-1 miss without rule_id" + + findings = _quick_scan(target) + run = BenchmarkRun(benchmark_run_id="split-sink", root=Path("/tmp/split-sink"), manifest=manifest) + result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None) + missed = {(miss.path, miss.cwe, miss.rule_id) for miss in result.misses} + + for expected in planted: + assert expected.line is not None + assert expected.sink + source = (target / expected.path).read_text() + lines = source.splitlines() + sink_line = lines[expected.line - 1] + assert expected.sink.lower() in sink_line.lower(), f"{name} planted sink {expected.sink!r} missing on line {expected.line}" + prior = "\n".join(lines[: expected.line - 1]).lower() + assert any(token in prior for token in ("query", "sql")), f"{name} must assign the payload before the sink line" + on_line = [finding for finding in findings if finding.path == expected.path and finding.line == expected.line] + assert on_line == [], f"{name} stage-1 hit the planted split-sink on {expected.path}:{expected.line}: {on_line}" + assert (expected.path, expected.cwe, None) in missed, ( + f"{name} evaluate_benchmark must record a miss for {expected.path}:{expected.line}" + ) + + +def test_split_sink_fixtures_are_outside_stage1_gate_corpus() -> None: + listed = {name for names in LANGUAGE_MANIFESTS.values() for name in names} + leaked = [name for name in SPLIT_SINK_MANIFESTS if name in listed] + assert leaked == [], "split-sink fixtures must not join LANGUAGE_MANIFESTS (stage-1 90% smoke gate)"