diff --git a/benchmarks/measurements/2026-09-21-partition-gap-blast-radius.json b/benchmarks/measurements/2026-09-21-partition-gap-blast-radius.json new file mode 100644 index 0000000..81e40a0 --- /dev/null +++ b/benchmarks/measurements/2026-09-21-partition-gap-blast-radius.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "experiment": "partition-gap-blast-radius-v1", + "date": "2026-09-21", + "task": "pre-push-safety-net 13.4", + "claim_scope": "Why a multi-language repository completes no question, in two parts. Measured on Paid Memberships Pro.", + "part_one_done": { + "change": "cross_partition_semantics_unresolved moved from GRAPH_INTEGRITY_GAPS to DECLARED_EXCLUSION_GAPS", + "argument": "Membership of the integrity set should turn on whether a gap can make a reported answer WRONG, not on whether it cost coverage. An unparsed file may hold the sanitizer that would have cleared a reported flow, so it belongs. A flow this tool cannot follow from one language into another can hide a finding and never invent one, because a PHP sink is sanitised in PHP.", + "what_it_touches": "Two places, both already honouring the distinction: the outcome demotion in scan.py and _blocking_degradations in the push policy, which exempts declared exclusions. The reason stays published in the scope's unresolved boundaries either way, so nothing is hidden.", + "correction": "The task note written earlier today said NOT to reclassify, on the grounds that it would claim cross-language flows are out of scope rather than unresolved. That was a misreading of the class: both classes are added to unresolved_boundaries, and membership decides only whether every answer in the repository is demoted." + }, + "part_two_open": { + "symptom": "PMPro still completes 0 of 127 questions after the reclassification, while establishing 19 findings", + "cause": "partition_file_census_incomplete, which is still an integrity gap and still fires", + "the_missing_files": [ + "js/blocks/blocks.build.js", + "js/jsapi.js", + "js/select2.min.js", + "webpack.config.js" + ], + "reading": "The JAVASCRIPT partition expected 36 files and its graph holds 32. The four absent ones are a webpack build output, a minified third-party library, a build config and one source file. The PHP partition is intact. So a plugin's bundled select2.min.js costs the entire PHP analysis its admissibility: 4,425 PHP regions demoted by four JavaScript files.", + "why_the_gap_is_right_and_its_blast_radius_is_not": "A file expected but missing from a graph could hold a sanitizer, so answers drawn from THAT partition may be wrong and demoting them is conservative. A missing JavaScript file cannot make a PHP answer wrong. This is the same over-broad blast radius as part one and the third instance of one principle in this thread: a gap should reach the questions it can actually affect.", + "fix_direction": "Scope a partition's census gap to that partition's questions. The language is known where the gap is raised, and path-to-language is already computed and used to dispatch every request, so the data exists; it is not threaded to the demotion.", + "second_order": "Three of the four files are build output or minified vendored code. The JavaScript vendored list holds only node_modules/, so a minified library under js/ is treated as first-party. Excluding build artefacts would remove this instance without addressing the blast radius." + }, + "measured_after_part_one": { + "questions_completed": 0, + "questions_total": 127, + "findings_total": 19, + "outcome_status": { + "unresolved": 127 + }, + "degradations": [ + "cross_partition_semantics_unresolved", + "partition_file_census_incomplete" + ] + }, + "limits": [ + "One multi-language subject. A single-language PHP repository would isolate part one's effect and has not been measured.", + "Part one is verified by argument and by the test suite, not yet by a subject that completes because of it." + ] +} diff --git a/benchmarks/push/family_census.py b/benchmarks/push/family_census.py index 64637cc..8bbfa79 100644 --- a/benchmarks/push/family_census.py +++ b/benchmarks/push/family_census.py @@ -83,6 +83,11 @@ def main() -> int: "query_seconds": result.query_seconds, "hook_table_hooks": len(hooks), "degradations": [str(d.get("reason")) for d in result.degradations], + # The reason alone cannot be acted on. A partition census that reports files missing carries WHICH + # files, and without them the gap is a label rather than a lead. + "degradation_detail": [ + {k: (v[:20] if isinstance(v, list) else v) for k, v in d.items() if k != "stage"} for d in result.degradations + ], "questions_total": len(result.question_outcomes), "questions_completed": sum(1 for q in result.question_outcomes if q.status == "completed"), "findings_total": len(result.findings), diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 1610967..cac505a 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -570,6 +570,17 @@ def _scan_repository_impl( # Reading the graph failed or produced something incomplete. An answer drawn from it cannot be # trusted, so a completed outcome is demoted. +# +# The test for membership is whether the gap could make a reported answer WRONG, not whether it cost +# coverage. An unparsed file may hold the sanitizer that would have cleared a flow this scan reports, +# so `files_unparsed` belongs here and demoting is the conservative reading. A missing file the +# partition census expected is the same shape. That is why `cross_partition_semantics_unresolved` is +# NOT here: a flow this tool cannot follow from one language into another can hide a finding, never +# invent one, because a PHP sink is sanitised in PHP. Demoting on it protected nothing and cost +# everything -- measured 2026-09-21, where a WordPress plugin with 36 regions of admin JavaScript +# completed none of 520 questions while establishing 43 findings, and admission requires a completed +# outcome. Every multi-language repository was unadmittable by construction, which is every plugin +# this product was built for. GRAPH_INTEGRITY_GAPS = frozenset( { "files_unparsed", @@ -577,7 +588,6 @@ GRAPH_INTEGRITY_GAPS = frozenset( "partition_file_census_incomplete", "partition_file_census_unavailable", "cpg_sharded", - "cross_partition_semantics_unresolved", "symlink_context_unresolved", "frontend_unsupported", "source_unreadable", @@ -588,7 +598,14 @@ GRAPH_INTEGRITY_GAPS = frozenset( # Declared, intentional exclusions. The exclusion stays physical and stays reported; it is not # evidence that any particular first-party answer is wrong. Treating it as one demoted every # answer on every real repository, because every real repository excludes its dependencies. -DECLARED_EXCLUSION_GAPS = frozenset({"vendor_semantics_unresolved"}) +# Declared, intentional exclusions. The exclusion stays physical and stays reported -- both classes are +# added to the scope's unresolved boundaries, so a declared gap is still published as unresolved; the class +# decides only whether it demotes every answer in the repository. +# +# `cross_partition_semantics_unresolved` is one of these: analysing each language separately, and not +# modelling a flow that leaves one for another, is a deliberate scope choice of exactly the kind vendor +# exclusion is. It was in the integrity set until 2026-09-21 and the reason for moving it is above. +DECLARED_EXCLUSION_GAPS = frozenset({"vendor_semantics_unresolved", "cross_partition_semantics_unresolved"}) def _identity(unit: str, region: ScanRegion, family: str) -> QuestionIdentity: