diff --git a/.kiro/specs/contributor-scan/frontend-retention-prerequisite.md b/.kiro/specs/contributor-scan/frontend-retention-prerequisite.md index d893ee3..5e0ae4e 100644 --- a/.kiro/specs/contributor-scan/frontend-retention-prerequisite.md +++ b/.kiro/specs/contributor-scan/frontend-retention-prerequisite.md @@ -1,6 +1,6 @@ # Frontend input retention prerequisite -Date: 2026-09-13. Status: proposed prerequisite, awaiting maintainer review. +Date: 2026-09-13. Status: approved by the maintainer ("approved"); task 2.18 verified complete after independent review. Owner: contributor-scan. Blocks pre-push-safety-net task 3.4. ## Confirmed problem @@ -26,7 +26,7 @@ Failure evidence is retained in An independent Kiro debug investigation returned `STOP_FOR_HUMAN`: a foundation prerequisite is missing from the approved downstream implementation sequence. -## Proposed prerequisite for approval +## Approved prerequisite Establish a reproducible frontend input policy that retains explicitly supplied first-party JavaScript source, including test directories and filename patterns. @@ -52,5 +52,6 @@ ranker candidates. Report actual frontend omissions as incomplete coverage. 4. Rebuild the image and rerun `ops/smoke_partitions.py`, then independently review and verify pre-push task 3.4 before checking it off or starting dependent work. -This proposal changes the dependency work required before task 3.4 can complete. -It does not approve a new engine version, a dependency patch or capability admission. +The maintainer approved this prerequisite and its dependency adaptation on 2026-09-13. +Task 2.18 owns reproducible implementation and verification; downstream task 3.4 +remains incomplete until its own review passes. Capability admission remains later work. diff --git a/.kiro/specs/contributor-scan/frontend-retention-verification.md b/.kiro/specs/contributor-scan/frontend-retention-verification.md new file mode 100644 index 0000000..d1f587a --- /dev/null +++ b/.kiro/specs/contributor-scan/frontend-retention-verification.md @@ -0,0 +1,45 @@ +# Frontend retention verification + +Date: 2026-09-13. Task: contributor-scan 2.18. Status: VERIFIED (TASK), independent review APPROVED. + +The maintainer approved the frontend-retention prerequisite after real graph +inspection showed that supplied JavaScript tests were missing. Verification must +establish retained source paths and method nodes, vendor exclusion, unchanged +ordinary engine behavior and explicit adaptation identity. + +The approved implementation adapts the two pinned upstream filters, preserving +original source bytes and paths. The existing ranker still marks tests unshipped; +the frontend inclusion policy does not select or admit findings. The installed +manifest distinguishes this adaptation from stock Joern 4.0.625 for future graph +compatibility checks. + +Completion requires independent review and a passing packaged census, including +the test-directory fixture and four filename suffixes. A test-disabled control +must reproduce the former omission. Native PHP/JavaScript graph and change-context +regressions must pass before downstream task 3.4 resumes. + +## Evidence + +`benchmarks/measurements/2026-09-13-frontend-retention-smoke.json` retains the +installed manifest, source/output integrity checks and complete runtime results. +The graph includes all six JavaScript files (255 source bytes) and two PHP files +(97 bytes), with exact test method witnesses and no vendor nodes. Disabling the +policy reproduces the original test omission with exit 1. The mixed scan uses +one selected question across both supported frontends and names unsupported Go. + +The PHP and JavaScript cross-file change-context regressions both complete with +one finding at the unchanged helper. The shared lab allowance is 900 seconds; +the mixed runtime took 260.93 seconds while concurrent checks were running. These +are controlled runtime checks, not hook latency measurements. + +The independent full suite passed 1,141 tests with nine skipped in 125.68 seconds. +Ruff, formatting (229 files), mypy (109 files) and diff checks passed. Independent +installer checks verified an offline idempotent skip and corruption rejection. +The original-backend RED reproduction is retained and explicitly distinguished +from the implementer's initial reported failing run. + +One backend comment changed after packaging; same-parser Python AST equality +proves no executable difference. Remaining listed modules match byte-for-byte; +installed dependency hashes match the manifest. + +Task 2.18 is complete. Pre-push task 3.4 requires its own final review and verification. diff --git a/.kiro/specs/contributor-scan/spec.json b/.kiro/specs/contributor-scan/spec.json index 6bf7e1a..40810dc 100644 --- a/.kiro/specs/contributor-scan/spec.json +++ b/.kiro/specs/contributor-scan/spec.json @@ -1,7 +1,7 @@ { "feature_name": "contributor-scan", "created_at": "2026-09-08T20:00:00Z", - "updated_at": "2026-09-13T09:04:09.994054+00:00", + "updated_at": "2026-09-13T09:29:15.806475+00:00", "language": "en", "phase": "implementation", "approvals": { @@ -35,8 +35,24 @@ "date": "2026-09-13", "document": "frontend-retention-prerequisite.md", "topic": "Retain declared first-party JavaScript tests through ASTGen and jssrc2cpg; prerequisite for pre-push-safety-net 3.4.", - "approved": false, - "ready_for_implementation": false + "approved": true, + "ready_for_implementation": true, + "approval_source": "approved", + "task": "2.18" } - ] + ], + "approval_history": [ + { + "at": "2026-09-13T09:17:01.788332+00:00", + "source": "approved", + "action": "Approved frontend-retention prerequisite, task 2.18, including reproducible dependency adaptation; resume pre-push-safety-net 3.4 after verification." + } + ], + "implementation_verification": { + "2.18": { + "status": "VERIFIED", + "review": "APPROVED", + "record": "frontend-retention-verification.md" + } + } } diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 8fbf041..efe60a5 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -333,6 +333,15 @@ - _Requirements: 5.1, 5.2, 5.5_ - _Depends: 2.11_ +- [x] 2.18 Retain declared first-party JavaScript source through the frontend + - Implement the approved `frontend-retention-prerequisite.md` with versioned source/checksum pins and a reproducible dependency adaptation across ASTGen and jssrc2cpg. + - Preserve source paths, bytes and imports; keep declared vendor sources excluded and tests unshipped. Report frontend omissions explicitly. + - Done when packaged file/method census retains test directories and filename suffixes, vendor witnesses remain absent, and existing native PHP/JavaScript/context/deadline regressions pass. + - _Boundary:_ contributor-scan engine integration and packaging; no new ranker, source IR or detector semantics. + - _Acceptance:_ approved frontend-retention-prerequisite.md acceptance 1–4; supports pre-push-safety-net requirements 2.2, 2.3, 7.3, 8.2, 8.4. + - _Approval:_ maintainer "approved", 2026-09-13; prerequisite for pre-push-safety-net task 3.4. + - _Verified:_ independent review APPROVED; 1,141 tests passed, nine skipped, static checks clean; real PHP/JavaScript test nodes retained and disabled-policy control reproduces omission. See `frontend-retention-verification.md`. + ## Group 3 — Ship it - [ ] 3.1 Contributor output @@ -1186,10 +1195,9 @@ Three requirements amendments this group argues for are drafted at the end of `r unapproved: what a report may repeat, a family stating a class it cannot decide, and Req 4.1 requiring a checkout that can actually demonstrate detection. -## Proposed prerequisite awaiting review — 2026-09-13 +## Approved frontend prerequisite — 2026-09-13 `frontend-retention-prerequisite.md` records a confirmed dependency gap blocking pre-push-safety-net task 3.4: pinned JavaScript frontend filters remove declared -first-party tests. Its dependency adaptation and acceptance checks are proposed, -not added to the approved executable task list. Review this prerequisite before -resuming downstream partition completion. +first-party tests. The maintainer approved its dependency adaptation and acceptance checks; +task 2.18 now implements it before downstream partition completion. diff --git a/.kiro/specs/pre-push-safety-net/spec.json b/.kiro/specs/pre-push-safety-net/spec.json index 47ae9aa..fe8bb3d 100644 --- a/.kiro/specs/pre-push-safety-net/spec.json +++ b/.kiro/specs/pre-push-safety-net/spec.json @@ -1,7 +1,7 @@ { "feature_name": "pre-push-safety-net", "created_at": "2026-09-12T15:35:00Z", - "updated_at": "2026-09-13T09:04:09.984662+00:00", + "updated_at": "2026-09-13T09:17:01.788332+00:00", "language": "en", "phase": "implementation", "approvals": { @@ -50,6 +50,11 @@ "at": "2026-09-13T07:48:02.155687Z", "source": "continue with the task", "action": "Authorized continuing group 3 (3.1\u20133.4) after the group 3 next-step summary. Independent task review and fresh verification remain required." + }, + { + "at": "2026-09-13T09:17:01.788332+00:00", + "source": "approved", + "action": "Approved contributor-scan frontend retention prerequisite; implement it and resume task 3.4 review and verification." } ], "task_review": { @@ -87,7 +92,7 @@ "task": "3.4", "reason": "Pinned JavaScript frontend silently excludes first-party tests; missing contributor-scan prerequisite.", "record": "../contributor-scan/frontend-retention-prerequisite.md", - "next_action": "review_prerequisite" + "next_action": "implement_approved_contributor_scan_2.18" } ] } diff --git a/Dockerfile b/Dockerfile index b30f8d9..6b70bad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,10 @@ RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \ fi +COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py +RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \ + && rm /tmp/frontend-retention-install.py + WORKDIR /app COPY pyproject.toml README.md ./ COPY src ./src diff --git a/benchmarks/measurements/2026-09-13-frontend-retention-smoke.json b/benchmarks/measurements/2026-09-13-frontend-retention-smoke.json new file mode 100644 index 0000000..1f85772 --- /dev/null +++ b/benchmarks/measurements/2026-09-13-frontend-retention-smoke.json @@ -0,0 +1,942 @@ +{ + "recorded_at": "2026-09-13T09:29:15.806475+00:00", + "task": "contributor-scan 2.18", + "verification": "VERIFIED", + "claim_type": "TASK", + "image_id": "sha256:85647aff62d771b22c2541895b780e0b0d6666bd0701ad0e6c518a5c6e1f64fe", + "build_exit_code": 0, + "mixed_smoke_exit_code": 0, + "context_smoke_exit_code": 0, + "disabled_control_exit_code": 1, + "disabled_control_expected": true, + "package": { + "hashes": { + "openultrasast.cpg.backend": "a53dc5f1170327a09f5501bef9a9b9c15c02e3efde9a352a15ec8fc147cb94bf", + "openultrasast.model.scan": "ad6f134c8fae43aa102b61d99f273b467415193d0b27fb634cae2586cb3c43ae", + "openultrasast.model.partitions": "c91970dbf536d4811e1980415a37e80348f38fbf9284ba469dfbac41c8215ac6", + "openultrasast.model.layout": "50a1a163459f8802037faae750bc1f99a0ab81912b571b40303675590794d27c", + "openultrasast.preprocess": "0d0b1f22fd2024be3e50dc1a9ff947eed7f8f60f67644277e344b83697aaefc8", + "openultrasast.cli": "79608d4a74bcceb5bd41b149cf806d25c4f71d8b3201d710217395941169bdc2" + }, + "manifest": { + "revision": "ousast-frontend-retention-v1", + "builder_sha256": "21be57123922fa83ab3a4a3c684f7ddd8bf4ffc79dfb8379226ea72d2b1cac02", + "policy": "OUSAST_INCLUDE_TESTS=1", + "sources": { + "astgen.tar.gz": [ + "https://codeload.github.com/joernio/astgen-monorepo/tar.gz/refs/tags/javascript-astgen/v3.50.1", + "8d9728dca8eab694a0f07bcd7a1c9a88368cb7bd354fc19c2ee8ca8611ff869a" + ], + "AstGenRunner.scala": [ + "https://raw.githubusercontent.com/joernio/joern/v4.0.625/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala", + "9963f949360d7e17baed1bd8875b717b2855745cf7260eef8a87d17533a54268" + ], + "bun.zip": [ + "https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip", + "36368faef7527875d5ffa52e53cd48021741f2a83eb6208a8dd64068d422a913" + ] + }, + "original_jar_sha256": "49fe488a79cc52b4e1d2e056143ec414f59e0007ed25f054c5fa7ae4aa34dff3", + "adapted_jar_sha256": "7b3a91a695abcd98652e83a4dfdbfd7c3be63b085e1bdfac0aae3139a714f1be", + "astgen_sha256": "eb1b01b2244e7dea4388716240bc0c1c1b66ff6de8cd1dfda30d3855627083e4", + "compiled_entries": [ + "io/joern/jssrc2cpg/utils/AstGenRunner$$anon$1.class", + "io/joern/jssrc2cpg/utils/AstGenRunner$.class", + "io/joern/jssrc2cpg/utils/AstGenRunner$astGenMetaData$.class", + "io/joern/jssrc2cpg/utils/AstGenRunner.class", + "io/joern/jssrc2cpg/utils/AstGenRunner.tasty" + ] + } + }, + "integrity": { + "manifest_outputs_verified": true, + "backend_ast_sha256": "0e3ebe0a1de7d45bbec5bf1e86904c9365a9e87dcfc07ea3c9da9fa7e3e40b8c", + "same_parser_backend_ast_equal": true + }, + "current_source_hashes": { + "openultrasast.cpg.backend": "f321ae8f6d3fbf5efc37a5de31453db0bb183a2651fe4650720f28f986a1dd78", + "openultrasast.model.scan": "ad6f134c8fae43aa102b61d99f273b467415193d0b27fb634cae2586cb3c43ae", + "openultrasast.model.partitions": "c91970dbf536d4811e1980415a37e80348f38fbf9284ba469dfbac41c8215ac6", + "openultrasast.model.layout": "50a1a163459f8802037faae750bc1f99a0ab81912b571b40303675590794d27c", + "openultrasast.preprocess": "0d0b1f22fd2024be3e50dc1a9ff947eed7f8f60f67644277e344b83697aaefc8", + "openultrasast.cli": "79608d4a74bcceb5bd41b149cf806d25c4f71d8b3201d710217395941169bdc2" + }, + "source_difference": "Backend comment-only clarification after image build; Python AST verified identical using the same local parser for both files. Other listed modules match byte-for-byte.", + "script_hashes": { + "ops/smoke_partitions.py": "113d1fe5b129506b80f445b17d44d7348f2dac0e11e11b569835ded5efbd309a", + "ops/smoke_context.py": "4d515258c01e44e2a30ee71f8de8de2df06c88dac2e0ded6f2904f7682e347d9" + }, + "mixed_runtime": { + "purpose": "Real mixed PHP/JavaScript graph census and one global ranker budget; no admission or latency claim", + "shared_budget_seconds": 900, + "include_tests": true, + "inputs": { + "src/api.php": { + "bytes": 49, + "sha256": "47ee82519e90f5ce2c845035f168bc4eaa4b0cc6e46cfc68626b8144e20b8f72" + }, + "server/api.js": { + "bytes": 51, + "sha256": "a9f1b897b9361d0d30250f82aebec987cfd99e66308c9ea889d5b6211e3cad5e" + }, + "tests/probe.php": { + "bytes": 48, + "sha256": "93c25992f2b709d4f222d88573a004dfb4e65239ce9a3afc2301f629b148c03d" + }, + "tests/probe.js": { + "bytes": 41, + "sha256": "182f7cb37514b7fc6afbba045a9d1b3ca5dc297c35bb98b9f34c18cfd02864f5" + }, + "vendor/library.php": { + "bytes": 55, + "sha256": "dd2aa2514d7c2dffdf4cfd238d7b795c2cd7bb6be086054b0b6d2278bd2c9599" + }, + "node_modules/library.js": { + "bytes": 46, + "sha256": "5e42f2831e2e32d8c19df33ba98fbda879495b74bd6d7e0b6ff45a077674f398" + }, + "unsupported.go": { + "bytes": 36, + "sha256": "ecf7a397251f1fea2e5992bdc3b0581274c20eb83f0ac403f6b7babe7d7b6b5b" + }, + "checks/probe.test.js": { + "bytes": 41, + "sha256": "182f7cb37514b7fc6afbba045a9d1b3ca5dc297c35bb98b9f34c18cfd02864f5" + }, + "checks/probe.spec.js": { + "bytes": 41, + "sha256": "a2677f51273a325b802a4a628ceb54f0bcea533a672c8997c0439d6ea206de37" + }, + "checks/probe.mock.js": { + "bytes": 41, + "sha256": "bc14cd012a0d2c34a3605ce21a009b7fd335bdf248f81aec3462b506929eabd5" + }, + "checks/probe.e2e.js": { + "bytes": 40, + "sha256": "14884e5a7d6c909f30addd8385bb82f6ea3751bb126192052dabe0eedfb42400" + } + }, + "graph_witnesses": [ + { + "frontend": "javascript", + "input_root": "/tmp/ousast-input-j_va8nqm", + "graph_witness": { + "files": [ + "checks/probe.e2e.js", + "checks/probe.mock.js", + "checks/probe.spec.js", + "checks/probe.test.js", + "server/api.js", + "tests/probe.js" + ], + "methods": [ + "js_e2e_marker", + ":program", + "js_mock_marker", + ":program", + "js_spec_marker", + ":program", + "js_test_marker", + ":program", + "js_handler", + ":program", + "js_test_marker", + ":program", + ".assignment", + "eval", + ".fieldAccess" + ] + } + }, + { + "frontend": "php", + "input_root": "/tmp/ousast-input-jr3nb1ie", + "graph_witness": { + "files": [ + "src/api.php", + "tests/probe.php", + "" + ], + "methods": [ + "", + "php_handler", + "", + "php_test_marker", + "eval" + ] + } + } + ], + "result": { + "findings": [ + { + "site": "src/api.php:2:php_handler", + "family": "injection", + "rung": "model_entailed", + "witness": "$req -> eval($req) in src/api.php (line 2, 2 steps)", + "contradiction": "", + "reached_from": 1 + } + ], + "by_rung": { + "suspicion": 0, + "model_corroborated": 0, + "model_entailed": 1, + "execution_confirmed": 0 + }, + "regions_scanned": 1, + "regions_unjudged": 3, + "regions_unasked": 0, + "model_calls": 0, + "cost_usd": 0.0, + "seconds": 260.93, + "build_seconds": 173.97, + "query_seconds": 43.9, + "query_seconds_by_kind": { + "evidence": 42.96, + "taint": 43.9 + }, + "degradations": [ + { + "stage": "model", + "reason": "cross_partition_semantics_unresolved" + }, + { + "stage": "model", + "reason": "frontend_unsupported" + }, + { + "stage": "model", + "reason": "vendor_semantics_unresolved" + }, + { + "stage": "model", + "reason": "regions_truncated", + "examined": 1, + "total": 4 + } + ], + "tiers": [ + [ + "src/api.php", + "php_handler", + "injection", + 4 + ] + ], + "tier_counts": { + "4": 1 + }, + "requests_pruned": 0, + "partitions": [ + { + "language": "go", + "frontend": null, + "paths": [ + "unsupported.go" + ], + "source_bytes": 0, + "status": "unsupported", + "boundaries": [ + "frontend_unsupported" + ], + "runtime": "unspecified", + "excluded_paths": [ + "node_modules/", + "vendor/" + ], + "unshipped_paths": [] + }, + { + "language": "javascript", + "frontend": "javascript", + "paths": [ + "checks/probe.e2e.js", + "checks/probe.mock.js", + "checks/probe.spec.js", + "checks/probe.test.js", + "server/api.js", + "tests/probe.js" + ], + "source_bytes": 255, + "status": "built", + "boundaries": [ + "runtime_unspecified" + ], + "runtime": "unspecified", + "excluded_paths": [ + "node_modules/", + "vendor/" + ], + "unshipped_paths": [ + "tests/probe.js" + ] + }, + { + "language": "php", + "frontend": "php", + "paths": [ + "src/api.php", + "tests/probe.php" + ], + "source_bytes": 97, + "status": "built", + "boundaries": [], + "runtime": "unspecified", + "excluded_paths": [ + "node_modules/", + "vendor/" + ], + "unshipped_paths": [ + "tests/probe.php" + ] + } + ], + "change_context": null, + "scope": { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "static", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "partition-smoke", + "language": "php", + "path": "src/api.php", + "function": "php_handler", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=10.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"mysqli_prepare\",\"mysqli_stmt_bind_param\",\"pg_execute\",\"pg_prepare\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval($req)\",\"line\":2,\"method\":\"php_handler\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [ + { + "identity": { + "unit": "partition-smoke", + "language": "javascript", + "path": "server/api.js", + "function": "js_handler", + "family": "injection" + }, + "reason": "region_budget", + "evidence": [ + "static_rank=9.0", + "evidence_unknown" + ] + }, + { + "identity": { + "unit": "partition-smoke", + "language": "php", + "path": "tests/probe.php", + "function": "php_test_marker", + "family": "injection" + }, + "reason": "region_budget", + "evidence": [ + "static_rank=2.0", + "evidence_unknown" + ] + }, + { + "identity": { + "unit": "partition-smoke", + "language": "javascript", + "path": "tests/probe.js", + "function": "js_test_marker", + "family": "injection" + }, + "reason": "region_budget", + "evidence": [ + "static_rank=1.0", + "evidence_unknown" + ] + } + ], + "unresolved_boundaries": [ + "evidence_unknown:022bab405f3a4daa5ee1cbca6b54c24d052b3060f915b850bc1b592d2233999c", + "evidence_unknown:d867a0ae5d785b179a30fcef9efcdb8304281d53694a51c2da4180ef5069a53d", + "evidence_unknown:66aa438d134c37f7107503949179ce763fecaa08f0a1d00646ee8a5f446d88f7", + "cross_partition_semantics_unresolved", + "frontend_unsupported", + "vendor_semantics_unresolved" + ] + }, + "question_outcomes": [ + { + "identity": { + "unit": "partition-smoke", + "language": "php", + "path": "src/api.php", + "function": "php_handler", + "family": "injection" + }, + "status": "unresolved", + "reason": "graph_incomplete", + "raw_rows_json": "[{\"bound\":\"\",\"bounded\":false,\"inLabeledScope\":true,\"length\":2,\"paths\":1,\"sanitized\":false,\"sink\":\"eval($req)\",\"sinkArg0Literal\":false,\"sinkArity\":1,\"sinkFile\":\"src/api.php\",\"sinkLine\":\"2\",\"sinkMethod\":\"php_handler\",\"source\":\"$req\",\"sourceKind\":\"parameter\"}]" + } + ], + "family_coverage": [ + { + "unit": "partition-smoke", + "language": "javascript", + "family": "injection", + "selected": 0, + "completed": 0, + "unanswered": 0, + "deferred": 2, + "unsupported": 0 + }, + { + "unit": "partition-smoke", + "language": "php", + "family": "injection", + "selected": 1, + "completed": 0, + "unanswered": 1, + "deferred": 1, + "unsupported": 0 + } + ] + } + }, + "context_runtime": { + "purpose": "Committed removed-guard context reaches real ranked questions; not semantic novelty or admission proof", + "shared_budget_seconds": 900, + "languages": { + "php": { + "base_revision": "4932fac446d4bca0642694e9a89cffaa739a31d5", + "head_revision": "32b5ef5ffe6a64bb4924fc7006273b0b2cc5a805", + "source_bytes": 124, + "source_sha256": "e41333ddd3e341d08603a5cc6a3a9faedb442e6ef3b14627cd00516cdf12cedb", + "helper_bytes": 48, + "helper_sha256": "4eb14937fd2b027cc11bd5a79c5de38f572d9a5af8ee36a46d4a0e499817ae76", + "input_context": { + "base_revision": "4932fac446d4bca0642694e9a89cffaa739a31d5", + "head_revision": "32b5ef5ffe6a64bb4924fc7006273b0b2cc5a805", + "changed_paths": [ + "6170692e706870" + ], + "deleted_paths": [], + "spans": [ + { + "path": "6170692e706870", + "start_line": 4, + "end_line": 4, + "side": "base" + } + ], + "renames": [], + "declaration_paths": [], + "relationships": [], + "unresolved_boundaries": [], + "path_encoding": "filesystem-bytes-hex", + "line_correspondences": [ + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 1, + "base_end_line": 1, + "head_start_line": 1 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 2, + "base_end_line": 2, + "head_start_line": 2 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 3, + "base_end_line": 3, + "head_start_line": 3 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 5, + "base_end_line": 5, + "head_start_line": 4 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 6, + "base_end_line": 6, + "head_start_line": 5 + } + ] + }, + "result": { + "findings": [ + { + "site": "helper.php:2:helper", + "family": "injection", + "rung": "model_entailed", + "witness": "$req -> eval($value) in helper.php (line 2, 4 steps)", + "contradiction": "", + "reached_from": 1 + } + ], + "by_rung": { + "suspicion": 0, + "model_corroborated": 0, + "model_entailed": 1, + "execution_confirmed": 0 + }, + "regions_scanned": 1, + "regions_unjudged": 0, + "regions_unasked": 0, + "model_calls": 0, + "cost_usd": 0.0, + "seconds": 170.02, + "build_seconds": 66.97, + "query_seconds": 58.08, + "query_seconds_by_kind": { + "evidence": 44.86, + "taint": 58.08 + }, + "degradations": [], + "tiers": [ + [ + "api.php", + "handler", + "injection", + 4 + ] + ], + "tier_counts": { + "4": 1 + }, + "requests_pruned": 0, + "partitions": [ + { + "language": "php", + "frontend": "php", + "paths": [ + "api.php", + "helper.php" + ], + "source_bytes": 124, + "status": "built", + "boundaries": [], + "runtime": "unspecified", + "excluded_paths": [], + "unshipped_paths": [] + } + ], + "change_context": { + "base_revision": "4932fac446d4bca0642694e9a89cffaa739a31d5", + "head_revision": "32b5ef5ffe6a64bb4924fc7006273b0b2cc5a805", + "changed_paths": [ + "6170692e706870" + ], + "deleted_paths": [], + "spans": [ + { + "path": "6170692e706870", + "start_line": 4, + "end_line": 4, + "side": "base" + } + ], + "renames": [], + "declaration_paths": [], + "relationships": [ + { + "source": { + "unit": "context-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "target": { + "unit": "context-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "kind": "entry", + "evidence": [ + "base_span:api.php:4-4", + "unchanged_line:3;lexical_only", + "unchanged_line:4;lexical_only", + "unchanged_line:5;lexical_only" + ] + } + ], + "unresolved_boundaries": [], + "path_encoding": "filesystem-bytes-hex", + "line_correspondences": [ + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 1, + "base_end_line": 1, + "head_start_line": 1 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 2, + "base_end_line": 2, + "head_start_line": 2 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 3, + "base_end_line": 3, + "head_start_line": 3 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 5, + "base_end_line": 5, + "head_start_line": 4 + }, + { + "base_path": "6170692e706870", + "head_path": "6170692e706870", + "base_start_line": 6, + "base_end_line": 6, + "head_start_line": 5 + } + ] + }, + "scope": { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "context-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"mysqli_prepare\",\"mysqli_stmt_bind_param\",\"pg_execute\",\"pg_prepare\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval($value)\",\"line\":2,\"method\":\"helper\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5", + "change_sites={\"base_revision\":\"4932fac446d4bca0642694e9a89cffaa739a31d5\",\"changed_paths\":[\"6170692e706870\"],\"declaration_paths\":[],\"deleted_paths\":[],\"head_revision\":\"32b5ef5ffe6a64bb4924fc7006273b0b2cc5a805\",\"path_encoding\":\"filesystem-bytes-hex\"}", + "affected_relationship={\"evidence\":[\"base_span:api.php:4-4\",\"unchanged_line:3;lexical_only\",\"unchanged_line:4;lexical_only\",\"unchanged_line:5;lexical_only\"],\"kind\":\"entry\",\"source\":{\"family\":\"injection\",\"function\":\"handler\",\"language\":\"php\",\"path\":\"api.php\",\"unit\":\"context-smoke\"},\"target\":{\"family\":\"injection\",\"function\":\"handler\",\"language\":\"php\",\"path\":\"api.php\",\"unit\":\"context-smoke\"}}" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [], + "unresolved_boundaries": [] + }, + "question_outcomes": [ + { + "identity": { + "unit": "context-smoke", + "language": "php", + "path": "api.php", + "function": "handler", + "family": "injection" + }, + "status": "completed", + "reason": "query_answered_and_arbitrated", + "raw_rows_json": "[{\"bound\":\"\",\"bounded\":false,\"inLabeledScope\":true,\"length\":4,\"paths\":1,\"sanitized\":false,\"sink\":\"eval($value)\",\"sinkArg0Literal\":false,\"sinkArity\":1,\"sinkFile\":\"helper.php\",\"sinkLine\":\"2\",\"sinkMethod\":\"helper\",\"source\":\"$req\",\"sourceKind\":\"parameter\"}]" + } + ], + "family_coverage": [ + { + "unit": "context-smoke", + "language": "php", + "family": "injection", + "selected": 1, + "completed": 1, + "unanswered": 0, + "deferred": 0, + "unsupported": 0 + } + ] + } + }, + "javascript": { + "base_revision": "94507460b2d291e67648c1431971864b379614e2", + "head_revision": "0900dcb2aaf4f82b134080247493a2b418c5d14f", + "source_bytes": 133, + "source_sha256": "ee6b40b0c84f84cda86b2f5d7d118f39504bc7237450d30e5c189f6b3517c965", + "helper_bytes": 47, + "helper_sha256": "173055ccb4d1b0cb76378bcb674ac9944bad02f848bf20139923775b57e7634d", + "input_context": { + "base_revision": "94507460b2d291e67648c1431971864b379614e2", + "head_revision": "0900dcb2aaf4f82b134080247493a2b418c5d14f", + "changed_paths": [ + "6170692e6a73" + ], + "deleted_paths": [], + "spans": [ + { + "path": "6170692e6a73", + "start_line": 3, + "end_line": 3, + "side": "base" + } + ], + "renames": [], + "declaration_paths": [], + "relationships": [], + "unresolved_boundaries": [], + "path_encoding": "filesystem-bytes-hex", + "line_correspondences": [ + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 1, + "base_end_line": 1, + "head_start_line": 1 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 2, + "base_end_line": 2, + "head_start_line": 2 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 4, + "base_end_line": 4, + "head_start_line": 3 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 5, + "base_end_line": 5, + "head_start_line": 4 + } + ] + }, + "result": { + "findings": [ + { + "site": "helper.js:1:helper", + "family": "injection", + "rung": "model_entailed", + "witness": "req -> eval(value) in helper.js (line 1, 4 steps)", + "contradiction": "", + "reached_from": 1 + } + ], + "by_rung": { + "suspicion": 0, + "model_corroborated": 0, + "model_entailed": 1, + "execution_confirmed": 0 + }, + "regions_scanned": 1, + "regions_unjudged": 0, + "regions_unasked": 0, + "model_calls": 0, + "cost_usd": 0.0, + "seconds": 106.45, + "build_seconds": 37.91, + "query_seconds": 24.29, + "query_seconds_by_kind": { + "evidence": 44.23, + "taint": 24.29 + }, + "degradations": [], + "tiers": [ + [ + "api.js", + "handler", + "injection", + 4 + ] + ], + "tier_counts": { + "4": 1 + }, + "requests_pruned": 0, + "partitions": [ + { + "language": "javascript", + "frontend": "javascript", + "paths": [ + "api.js", + "helper.js" + ], + "source_bytes": 133, + "status": "built", + "boundaries": [ + "runtime_unspecified" + ], + "runtime": "unspecified", + "excluded_paths": [], + "unshipped_paths": [] + } + ], + "change_context": { + "base_revision": "94507460b2d291e67648c1431971864b379614e2", + "head_revision": "0900dcb2aaf4f82b134080247493a2b418c5d14f", + "changed_paths": [ + "6170692e6a73" + ], + "deleted_paths": [], + "spans": [ + { + "path": "6170692e6a73", + "start_line": 3, + "end_line": 3, + "side": "base" + } + ], + "renames": [], + "declaration_paths": [], + "relationships": [ + { + "source": { + "unit": "context-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "injection" + }, + "target": { + "unit": "context-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "injection" + }, + "kind": "entry", + "evidence": [ + "base_span:api.js:3-3", + "unchanged_line:2;lexical_only", + "unchanged_line:3;lexical_only", + "unchanged_line:4;lexical_only" + ] + } + ], + "unresolved_boundaries": [], + "path_encoding": "filesystem-bytes-hex", + "line_correspondences": [ + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 1, + "base_end_line": 1, + "head_start_line": 1 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 2, + "base_end_line": 2, + "head_start_line": 2 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 4, + "base_end_line": 4, + "head_start_line": 3 + }, + { + "base_path": "6170692e6a73", + "head_path": "6170692e6a73", + "base_start_line": 5, + "base_end_line": 5, + "head_start_line": 4 + } + ] + }, + "scope": { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "context-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"execute\",\"query\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval(value)\",\"line\":1,\"method\":\"helper\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5", + "change_sites={\"base_revision\":\"94507460b2d291e67648c1431971864b379614e2\",\"changed_paths\":[\"6170692e6a73\"],\"declaration_paths\":[],\"deleted_paths\":[],\"head_revision\":\"0900dcb2aaf4f82b134080247493a2b418c5d14f\",\"path_encoding\":\"filesystem-bytes-hex\"}", + "affected_relationship={\"evidence\":[\"base_span:api.js:3-3\",\"unchanged_line:2;lexical_only\",\"unchanged_line:3;lexical_only\",\"unchanged_line:4;lexical_only\"],\"kind\":\"entry\",\"source\":{\"family\":\"injection\",\"function\":\"handler\",\"language\":\"javascript\",\"path\":\"api.js\",\"unit\":\"context-smoke\"},\"target\":{\"family\":\"injection\",\"function\":\"handler\",\"language\":\"javascript\",\"path\":\"api.js\",\"unit\":\"context-smoke\"}}" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [], + "unresolved_boundaries": [] + }, + "question_outcomes": [ + { + "identity": { + "unit": "context-smoke", + "language": "javascript", + "path": "api.js", + "function": "handler", + "family": "injection" + }, + "status": "completed", + "reason": "query_answered_and_arbitrated", + "raw_rows_json": "[{\"bound\":\"\",\"bounded\":false,\"inLabeledScope\":true,\"length\":4,\"paths\":1,\"sanitized\":false,\"sink\":\"eval(value)\",\"sinkArg0Literal\":false,\"sinkArity\":1,\"sinkFile\":\"helper.js\",\"sinkLine\":\"1\",\"sinkMethod\":\"helper\",\"source\":\"req\",\"sourceKind\":\"parameter\"}]" + } + ], + "family_coverage": [ + { + "unit": "context-smoke", + "language": "javascript", + "family": "injection", + "selected": 1, + "completed": 1, + "unanswered": 0, + "deferred": 0, + "unsupported": 0 + } + ] + } + } + } + }, + "disabled_control_log": "{\"frontend\": \"javascript\", \"projected_inputs\": {\"tests/probe.js\": {\"bytes\": 41, \"sha256\": \"182f7cb37514b7fc6afbba045a9d1b3ca5dc297c35bb98b9f34c18cfd02864f5\"}, \"server/api.js\": {\"bytes\": 51, \"sha256\": \"a9f1b897b9361d0d30250f82aebec987cfd99e66308c9ea889d5b6211e3cad5e\"}, \"checks/probe.spec.js\": {\"bytes\": 41, \"sha256\": \"a2677f51273a325b802a4a628ceb54f0bcea533a672c8997c0439d6ea206de37\"}, \"checks/probe.e2e.js\": {\"bytes\": 40, \"sha256\": \"14884e5a7d6c909f30addd8385bb82f6ea3751bb126192052dabe0eedfb42400\"}, \"checks/probe.test.js\": {\"bytes\": 41, \"sha256\": \"182f7cb37514b7fc6afbba045a9d1b3ca5dc297c35bb98b9f34c18cfd02864f5\"}, \"checks/probe.mock.js\": {\"bytes\": 41, \"sha256\": \"bc14cd012a0d2c34a3605ce21a009b7fd335bdf248f81aec3462b506929eabd5\"}}}\n{\"frontend\": \"javascript\", \"graph_witness\": {\"files\": [\"server/api.js\"], \"methods\": [\"js_handler\", \":program\", \"eval\", \".fieldAccess\", \".assignment\"]}}\nTraceback (most recent call last):\n File \"\", line 138, in \n File \"\", line 107, in main\n File \"/app/src/openultrasast/model/scan.py\", line 172, in scan_repository\n result = _scan_repository_impl(\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/app/src/openultrasast/model/scan.py\", line 234, in _scan_repository_impl\n cpg = build_partitions(\n ^^^^^^^^^^^^^^^^^\n File \"/app/src/openultrasast/model/partitions.py\", line 222, in build_partitions\n built = build(scratch, frontend) if complete else None\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/app/src/openultrasast/model/scan.py\", line 236, in \n lambda source, language: _build(backend, source, language, execution_budget=execution_budget),\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/app/src/openultrasast/model/scan.py\", line 815, in _build\n return backend.build(root, language=language, exclude=excluded, execution_budget=execution_budget)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\", line 88, in build\nAssertionError: {'files': ['server/api.js'], 'methods': ['js_handler', ':program', 'eval', '.fieldAccess', '.assignment']}\n", + "astgen_control_log": "{'api.js': 26, 'tests/probe.js': 36, 'probe.test.js': 33, 'vendor/probe.js': 35}\nConverted AST for api.js to /tmp/tmpxfnr1il2/0/api.js.json\nConverted types for api.js to /tmp/tmpxfnr1il2/0/api.js.typemap\nConverted AST for probe.test.js to /tmp/tmpxfnr1il2/1/probe.test.js.json\nConverted AST for tests/probe.js to /tmp/tmpxfnr1il2/1/tests/probe.js.json\nConverted AST for api.js to /tmp/tmpxfnr1il2/1/api.js.json\nConverted types for api.js to /tmp/tmpxfnr1il2/1/api.js.typemap\nConverted types for probe.test.js to /tmp/tmpxfnr1il2/1/probe.test.js.typemap\nConverted types for tests/probe.js to /tmp/tmpxfnr1il2/1/tests/probe.js.typemap\n{\"0\": [\"api.js.json\"], \"1\": [\"api.js.json\", \"probe.test.js.json\", \"tests/probe.js.json\"]}\n", + "red_reproduction": "FF. [100%]\n=================================== FAILURES ===================================\n__________________ test_first_party_tests_enabled_by_backend ___________________\n\nmonkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7044587f6cd0>\n\n def test_first_party_tests_enabled_by_backend(monkeypatch):\n monkeypatch.setenv(\"OUSAST_INCLUDE_TESTS\", \"0\")\n> assert JoernBackend()._jvm_env()[\"OUSAST_INCLUDE_TESTS\"] == \"1\"\nE AssertionError: assert '0' == '1'\nE \nE - 1\nE + 0\n\ntests/test_frontend_retention.py:8: AssertionError\n__________________ test_retention_can_be_disabled_explicitly ___________________\n\n def test_retention_can_be_disabled_explicitly():\n> assert JoernBackend(include_tests=False)._jvm_env()[\"OUSAST_INCLUDE_TESTS\"] == \"0\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: JoernBackend.__init__() got an unexpected keyword argument 'include_tests'\n\ntests/test_frontend_retention.py:12: TypeError\n=========================== short test summary info ============================\nFAILED tests/test_frontend_retention.py::test_first_party_tests_enabled_by_backend\nFAILED tests/test_frontend_retention.py::test_retention_can_be_disabled_explicitly\n2 failed, 1 passed in 1.16s\n", + "red_reproduction_note": "Postimplementation reproduction against original committed backend; matches implementer-reported initial RED.", + "independent_review": "APPROVED", + "pytest": { + "passed": 1141, + "skipped": 9, + "seconds": 125.68, + "exit_code": 0 + }, + "static_checks": "Ruff,format229,mypy109,diffcheck pass", + "limits": "Prerequisite retention and controlled runtime verification; not hook latency, general detection quality or capability admission." +} diff --git a/ops/frontend-retention/README.md b/ops/frontend-retention/README.md new file mode 100644 index 0000000..c3d67ef --- /dev/null +++ b/ops/frontend-retention/README.md @@ -0,0 +1,36 @@ +# First-party JavaScript test retention + +This source adaptation implements contributor-scan task 2.18 for the pinned Linux +x86-64 Joern 4.0.625 installation. ASTGen 3.50.1 excludes test directories and +filenames; jssrc2cpg additionally excludes test suffixes. Both filters must accept +the explicit inclusion policy for a retained source file to reach the graph. + +`install.py` verifies pinned upstream source and tool archives and the original +jssrc2cpg jar, rebuilds ASTGen with its frozen dependency lockfile, and compiles +the adapted Scala source against Joern's shipped compiler and libraries. The +changes affect test filtering only. Original input paths, bytes and imports are +preserved. Vendor and user exclusions still apply. + +Docker and `ops/joern.py` run the same installer. A manual installation requires +Python 3.12 or later, Java 21, network access for the pinned build inputs, and a +pristine matching Joern installation: + +```bash +python3 ops/frontend-retention/install.py /opt/joern-cli +``` + +The installer writes `ousast-frontend-retention-v1/manifest.json` under the Joern +root, including builder, dependency and output hashes. A repeated invocation +verifies the installed outputs and needs no network. If the recipe or installed +outputs change, reinstall the pristine pinned engine before rebuilding; the +installer refuses to layer an unverified adaptation onto modified jars. + +The adapted frontend keeps stock behavior unless `OUSAST_INCLUDE_TESTS=1`. +`JoernBackend` explicitly enables it by default; `include_tests=False` supplies +the regression control. Direct frontend users must set the variable themselves. +Engine provenance must include the installed adaptation and effective policy: +the upstream version string alone cannot distinguish these graphs. + +After building the image, run `ops/smoke_partitions.py` as described in `ops/README.md`. +Its off control uses `OUSAST_SMOKE_INCLUDE_TESTS=0` and must fail the test-retention +assertion. First-party test retention does not imply security capability admission. diff --git a/ops/frontend-retention/install.py b/ops/frontend-retention/install.py new file mode 100644 index 0000000..35cb0b1 --- /dev/null +++ b/ops/frontend-retention/install.py @@ -0,0 +1,151 @@ +"""Build the approved test-retention adaptation from checksum-pinned upstream sources. + +Usage: python3 install.py /opt/joern-cli. Requires curl, unzip, Java 21 and Python3. +Only the two upstream input filters change. OUSAST_INCLUDE_TESTS=1 opts in; +unset/0 keeps upstream behavior. Sources, lockfile and build identity are retained. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +PINS = { + "astgen.tar.gz": ( + "https://codeload.github.com/joernio/astgen-monorepo/tar.gz/refs/tags/javascript-astgen/v3.50.1", + "8d9728dca8eab694a0f07bcd7a1c9a88368cb7bd354fc19c2ee8ca8611ff869a", + ), + "AstGenRunner.scala": ( + "https://raw.githubusercontent.com/joernio/joern/v4.0.625/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala", + "9963f949360d7e17baed1bd8875b717b2855745cf7260eef8a87d17533a54268", + ), + "bun.zip": ( + "https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip", + "36368faef7527875d5ffa52e53cd48021741f2a83eb6208a8dd64068d422a913", + ), +} +JAR_SHA256 = "49fe488a79cc52b4e1d2e056143ec414f59e0007ed25f054c5fa7ae4aa34dff3" + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def replace_once(path: Path, before: str, after: str) -> None: + source = path.read_text() + if source.count(before) != 1: + raise ValueError(f"Unexpected source shape: {path.name}") + path.write_text(source.replace(before, after)) + + +def build(root: Path) -> None: + jar = root / "frontends/jssrc2cpg/lib/io.joern.jssrc2cpg-4.0.625.jar" + installed = root / "ousast-frontend-retention-v1/manifest.json" + if installed.is_file(): + previous = json.loads(installed.read_text()) + if ( + previous.get("builder_sha256") == digest(Path(__file__)) + and previous.get("adapted_jar_sha256") == digest(jar) + and previous.get("astgen_sha256") == digest(root / "frontends/jssrc2cpg/bin/astgen/astgen-linux") + ): + print("Verified installed frontend-retention adaptation") + return + if digest(jar) != JAR_SHA256: + raise ValueError("Expected pristine Joern 4.0.625 jssrc2cpg jar") + with tempfile.TemporaryDirectory(prefix="ousast-frontend-build-") as temporary: + work = Path(temporary) + for name, (url, expected) in PINS.items(): + with urllib.request.urlopen(url) as response: + (work / name).write_bytes(response.read()) + if digest(work / name) != expected: + raise ValueError(f"Checksum mismatch: {name}") + with tarfile.open(work / "astgen.tar.gz") as archive: + archive.extractall(work, filter="data") + with zipfile.ZipFile(work / "bun.zip") as archive: + archive.extractall(work) + bun = work / "bun-linux-x64/bun" + bun.chmod(0o755) + astgen = work / "astgen-monorepo-javascript-astgen-v3.50.1/javascript-astgen" + file_utils = astgen / "src/FileUtils.ts" + replace_once( + file_utils, + "const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.map((d) => d.toLowerCase()))", + """const INCLUDE_TESTS = process.env.OUSAST_INCLUDE_TESTS === "1" +const TEST_DIRS = new Set(["test", "tests", "e2e", "e2e-beta", "cypress", "__tests__", "__mocks__"]) +const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.filter(d => !INCLUDE_TESTS || !TEST_DIRS.has(d)).map(d => d.toLowerCase())) +const IGNORE_FILE_PATTERN = INCLUDE_TESTS + ? new RegExp("(chunk-vendors|app~|conf|[.-]min|\\\\.d)\\\\.(js|jsx|cjs|mjs|xsjs|xsjslib|ts|tsx)$", "i") + : Defaults.IGNORE_FILE_PATTERN""", + ) + replace_once( + file_utils, + 'dirName.startsWith("__") ||', + '(dirName.startsWith("__") && !(INCLUDE_TESTS && TEST_DIRS.has(dirName.toLowerCase()))) ||', + ) + replace_once(file_utils, "Defaults.IGNORE_FILE_PATTERN.test(fileName)", "IGNORE_FILE_PATTERN.test(fileName)") + scala = work / "AstGenRunner.scala" + replace_once( + scala, + "lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(filePath))", + 'lazy val isIgnoredTest = !sys.env.get("OUSAST_INCLUDE_TESTS").contains("1") && IgnoredTestsRegex.exists(_.matches(filePath))', + ) + # Upstream stages embedded TS standard-library assets; retain that build recipe, + # selecting just the supported Linux x64 target instead of all release platforms. + script = astgen / "scripts/build-binaries.ts" + source = script.read_text() + start = source.index("const targets = [") + end = source.index("] as const", start) + len("] as const") + script.write_text( + source[:start] + 'const targets = [{target: "bun-linux-x64", outfile: "./astgen-linux-x64"}] as const' + source[end:] + ) + subprocess.run([str(bun), "install", "--frozen-lockfile"], cwd=astgen, check=True) + subprocess.run([str(bun), "scripts/sync-version.ts"], cwd=astgen, check=True) + subprocess.run([str(bun), "scripts/build-binaries.ts"], cwd=astgen, check=True) + classes = work / "classes" + classes.mkdir() + jars = sorted(set(root.glob("lib/*.jar")) | set(root.glob("frontends/jssrc2cpg/lib/*.jar"))) + classpath = ":".join(map(str, jars)) + subprocess.run( + ["java", "-cp", classpath, "dotty.tools.dotc.Main", "-classpath", classpath, "-d", str(classes), str(scala)], check=True + ) + outputs = sorted(classes.rglob("*")) + compiled = {path.relative_to(classes).as_posix(): path for path in outputs if path.is_file()} + if not compiled or any(not name.startswith("io/joern/jssrc2cpg/utils/AstGenRunner") for name in compiled): + raise ValueError("Compiler produced unexpected classes") + adapted = work / "adapted.jar" + with zipfile.ZipFile(jar) as original, zipfile.ZipFile(adapted, "w", zipfile.ZIP_DEFLATED) as destination: + for entry in original.infolist(): + if entry.filename not in compiled: + destination.writestr(entry, original.read(entry.filename)) + for name, path in compiled.items(): + destination.write(path, name) + provenance = root / "ousast-frontend-retention-v1" + provenance.mkdir(exist_ok=True) + for source_path in (file_utils, scala, script, astgen / "bun.lock"): + shutil.copy2(source_path, provenance / source_path.name) + binary = astgen / "astgen-linux-x64" + manifest = { + "revision": "ousast-frontend-retention-v1", + "builder_sha256": digest(Path(__file__)), + "policy": "OUSAST_INCLUDE_TESTS=1", + "sources": PINS, + "original_jar_sha256": JAR_SHA256, + "adapted_jar_sha256": digest(adapted), + "astgen_sha256": digest(binary), + "compiled_entries": sorted(compiled), + } + shutil.copy2(binary, root / "frontends/jssrc2cpg/bin/astgen/astgen-linux") + shutil.copy2(adapted, jar) + (provenance / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +if __name__ == "__main__": + build(Path(sys.argv[1]).resolve()) diff --git a/ops/joern.py b/ops/joern.py index d0387b4..065779c 100644 --- a/ops/joern.py +++ b/ops/joern.py @@ -40,7 +40,7 @@ STAMP = f"{PREFIX}/.joern-version" # PHP-Parser), so without `php` on PATH a PHP tree fails to build with an opaque "Process exited with code 1" # -- measured on this machine before the package was added. `php_frontend` is opt-in because most targets do # not need it and it pulls a second runtime. -PACKAGES = ["openjdk-21-jre-headless", "unzip"] +PACKAGES = ["openjdk-21-jre-headless", "unzip", "python3", "curl", "ca-certificates"] if host.data.get("php_frontend", False): PACKAGES.append("php-cli") @@ -101,3 +101,19 @@ server.shell( name="Verify joern-parse runs", commands=[f"{PREFIX}/joern-cli/joern-parse --help > /dev/null"], ) + +# The same checksum-pinned source adaptation as the packaged image. Installing it is +# explicit and fails on an unexpected engine; stock versions do not claim parity. +files.put( + name="Upload the versioned frontend-retention builder", + src="ops/frontend-retention/install.py", + dest=f"{PREFIX}/frontend-retention-install.py", + _sudo=True, +) +server.shell( + name="Build the approved frontend-retention adaptation", + commands=[ + f"python3 {PREFIX}/frontend-retention-install.py {PREFIX}/joern-cli", + ], + _sudo=True, +) diff --git a/ops/smoke_partitions.py b/ops/smoke_partitions.py index 38e08a5..f457957 100644 --- a/ops/smoke_partitions.py +++ b/ops/smoke_partitions.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import json +import os import sys import tempfile import time @@ -17,6 +18,7 @@ from openultrasast.model.scan import ScanBudget, scan_repository def main() -> None: + include_tests = os.environ.get("OUSAST_SMOKE_INCLUDE_TESTS", "1") == "1" deadline = ExecutionBudget(time.monotonic() + 900, 2.0) records: list[dict[str, object]] = [] with tempfile.TemporaryDirectory(prefix="ousast-partitions-smoke-") as temporary: @@ -43,6 +45,8 @@ def main() -> None: "node_modules/library.js": "function vendor_js_marker(req) { eval(req); }\n", "unsupported.go": "package example\nfunc Unmodeled() {}\n", } + suffix_tests = {f"checks/probe.{suffix}.js": f"js_{suffix}_marker" for suffix in ("test", "spec", "mock", "e2e")} + sources.update({path: f"function {method}() {{ return 42; }}\n" for path, method in suffix_tests.items()}) inputs = {} for filename, content in sources.items(): path = root / filename @@ -83,6 +87,9 @@ def main() -> None: assert any(str(path).endswith("server/api.js") for path in files), witness assert any(str(path).endswith("tests/probe.js") for path in files), witness assert "js_handler" in methods and "js_test_marker" in methods, witness + for path, method in suffix_tests.items(): + assert any(str(filename).endswith(path) for filename in files), witness + assert method in methods, witness assert "php_handler" not in methods, witness else: raise AssertionError(f"unexpected frontend: {language}") @@ -100,7 +107,7 @@ def main() -> None: result = scan_repository( root, regions, - backend=RecordingBackend(heap_mb=1024), + backend=RecordingBackend(heap_mb=1024, include_tests=include_tests), budget=ScanBudget(max_regions=1), execution_budget=deadline, ranking_mode="static", @@ -119,6 +126,7 @@ def main() -> None: report = { "purpose": "Real mixed PHP/JavaScript graph census and one global ranker budget; no admission or latency claim", "shared_budget_seconds": 900, + "include_tests": include_tests, "inputs": inputs, "graph_witnesses": records, "result": asdict(result), diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index eb4637f..6b8088f 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -270,6 +270,7 @@ class JoernBackend: """``joern-parse`` then ``joern --script``, both out of process and both timeout-bounded.""" runner: Runner | None = None + include_tests: bool = True # Explicit input policy for the versioned frontend adaptation. build_timeout: int = BUILD_TIMEOUT_SECONDS query_timeout: int = field(default_factory=lambda: _configured_timeout()) heap_mb: int = 0 # 0 means read the environment, then fall back to CPG_HEAP_MB @@ -367,18 +368,28 @@ class JoernBackend: cleanup=dispose, execution_diagnostics=lambda: tuple(self._diagnostics), ) - command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)] - completed = self._run(command, timeout=self.build_timeout, cwd=scratch) - unparsed = _unparsed_files(completed) - if completed is None or completed.returncode != 0 or not cpg_path.is_file(): - detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" - logger.warning("cpg build failed for %s: %s", root, detail) - retried = self._build_with_frontend(root, cpg_path, scratch, language) - if retried is None: - dispose() # a failed build's scratch is nobody's to sweep six hours later + if language.lower() in {"javascript", "typescript"}: + # Use the declared frontend rather than language autodetection. The versioned + # retention adaptation consumes the explicit include_tests subprocess policy. + direct = self._build_with_frontend(root, cpg_path, scratch, language, exclude) + if direct is None: + dispose() return None self._apply_overlays(cpg_path, scratch) - unparsed = retried + unparsed = direct + else: + command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)] + completed = self._run(command, timeout=self.build_timeout, cwd=scratch) + unparsed = _unparsed_files(completed) + if completed is None or completed.returncode != 0 or not cpg_path.is_file(): + detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" + logger.warning("cpg build failed for %s: %s", root, detail) + retried = self._build_with_frontend(root, cpg_path, scratch, language) + if retried is None: + dispose() # a failed build's scratch is nobody's to sweep six hours later + return None + self._apply_overlays(cpg_path, scratch) + unparsed = retried if unparsed: logger.warning("the frontend could not parse %d file(s) under %s: %s", len(unparsed), root, ", ".join(unparsed[:5])) return CpgResult( @@ -1004,6 +1015,7 @@ class JoernBackend: options survive. """ env = dict(os.environ) + env["OUSAST_INCLUDE_TESTS"] = "1" if self.include_tests else "0" env["JAVA_TOOL_OPTIONS"] = f"{env.get('JAVA_TOOL_OPTIONS', '').strip()} -Xmx{self._heap_mb()}m".strip() return env diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 623e44b..639f85c 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -1103,3 +1103,29 @@ def test_the_interpreter_must_read_the_frontend_parser_too(tmp_path: Path, monke backend = JoernBackend(runner=runner) assert backend.build(tmp_path / "repo", language="php") is None assert "php-parser.php" in backend.last_failure and "mounts the repository but not this install" in backend.last_failure + + +def test_javascript_partition_uses_declared_frontend_not_legacy_autodetection(tmp_path, monkeypatch): + from openultrasast.cpg.backend import JoernBackend + + (tmp_path / "tests").mkdir() + (tmp_path / "tests/probe.js").write_bytes(b"function first_party_test(x) { return x; }") + commands = [] + + def runner(command, **kwargs): + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + commands.append(command) + assert Path(command[0]).name == "jssrc2cpg" + Path(command[command.index("-o") + 1]).write_bytes(b"graph") + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}") + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + result = JoernBackend(runner=runner).build(tmp_path, language="javascript") + try: + assert result is not None + assert len(commands) == 1 + finally: + if result is not None: + result.cleanup() diff --git a/tests/test_cpg_budget.py b/tests/test_cpg_budget.py index fe27253..5ae98ac 100644 --- a/tests/test_cpg_budget.py +++ b/tests/test_cpg_budget.py @@ -168,6 +168,8 @@ def test_explicit_build_budget_is_retained_without_mutating_backend(tmp_path: Pa calls.append(kwargs["timeout"]) if "--output" in command: Path(command[command.index("--output") + 1]).write_bytes(b"graph fixture") + elif "-o" in command: + Path(command[command.index("-o") + 1]).write_bytes(b"graph fixture") return subprocess.CompletedProcess(command, 0, f'{BEGIN}\n{{"r":[]}}\n{END}', "") monkeypatch.setattr(module.shutil, "which", lambda name: name) diff --git a/tests/test_frontend_retention.py b/tests/test_frontend_retention.py new file mode 100644 index 0000000..744f0d8 --- /dev/null +++ b/tests/test_frontend_retention.py @@ -0,0 +1,31 @@ +"""Explicit frontend input policy must survive both subprocess paths.""" + +from openultrasast.cpg.backend import JoernBackend + + +def test_first_party_tests_enabled_by_backend(monkeypatch): + monkeypatch.setenv("OUSAST_INCLUDE_TESTS", "0") + assert JoernBackend()._jvm_env()["OUSAST_INCLUDE_TESTS"] == "1" + + +def test_retention_can_be_disabled_explicitly(): + assert JoernBackend(include_tests=False)._jvm_env()["OUSAST_INCLUDE_TESTS"] == "0" + + +def test_builder_rejects_unrecognized_dependency_before_download(tmp_path): + import importlib.util + from pathlib import Path + + import pytest + + spec = importlib.util.spec_from_file_location("retention_builder", Path("ops/frontend-retention/install.py")) + assert spec is not None and spec.loader is not None + builder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(builder) + jar = tmp_path / "frontends/jssrc2cpg/lib/io.joern.jssrc2cpg-4.0.625.jar" + jar.parent.mkdir(parents=True) + jar.write_bytes(b"different dependency") + with pytest.raises(ValueError, match="pristine Joern"): + builder.build(tmp_path) + assert jar.read_bytes() == b"different dependency" + assert not (tmp_path / "ousast-frontend-retention-v1").exists()