From e13ce35a7531fc67a54a3d1697b644066074054c Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 4 Oct 2026 10:34:50 +0200 Subject: [PATCH] engine: admit TypeScript (jssrc2cpg, JavaScript taint specs via one alias), .mts/.cts, mixed JS/TS builds one CPG Implemented with Codex. Not merged: wall-clock deadline tests failed at host load 47; to be re-checked against main at normal load. Co-Authored-By: Claude Opus 5.5 --- benchmarks/learn/build_features.py | 5 +- src/openultrasast/model/partitions.py | 63 +++++++++++++++++-------- src/openultrasast/model/regions.py | 3 +- src/openultrasast/model/specs.py | 7 ++- src/openultrasast/pairs.py | 4 +- src/openultrasast/plane/tasks/alerts.py | 6 ++- src/openultrasast/preprocess.py | 2 + src/openultrasast/semantic/facts.py | 16 ++++++- tests/test_cpg_backend.py | 30 ++++++++++++ tests/test_learn_build_features.py | 4 +- tests/test_model_partitions.py | 34 +++++++++++++ tests/test_model_specs.py | 23 +++++++++ tests/test_plane_loop.py | 12 +++++ 13 files changed, 175 insertions(+), 34 deletions(-) diff --git a/benchmarks/learn/build_features.py b/benchmarks/learn/build_features.py index fdc87eb..ae650b4 100644 --- a/benchmarks/learn/build_features.py +++ b/benchmarks/learn/build_features.py @@ -126,9 +126,8 @@ def resolve_scan(scans: Path, case: str, side: str) -> dict[str, Any] | None: def recorded_engine(language: str, findings: list, result: Mapping | None, repository: bool) -> Part: - # The language list describes standalone scans. Repository scans also analyze - # TypeScript; recorded function findings take precedence over that list. - if language not in ENGINE_LANGUAGES and not findings and not (repository and language == "typescript" and result is not None): + # Recorded findings take precedence over the shared engine language coverage. + if language not in ENGINE_LANGUAGES and not findings: return NONE state = engine_for(result) if state is not None: diff --git a/src/openultrasast/model/partitions.py b/src/openultrasast/model/partitions.py index d283bf6..93584ce 100644 --- a/src/openultrasast/model/partitions.py +++ b/src/openultrasast/model/partitions.py @@ -19,7 +19,7 @@ from .layout import is_test_path, is_vendored, layout_facts FRONTENDS = { "php": "php", "javascript": "javascript", - "typescript": "typescript", + "typescript": "javascript", "python": "python", "java": "java", "c": "c", @@ -109,14 +109,19 @@ class PartitionGraph: def _dispatch(self, kind: str, requests: Mapping[str, Mapping[str, object]], method: str) -> dict[str, list[object]]: answers: dict[str, list[object]] = {} - languages = dict.fromkeys(self.path_languages.get(str(params.get("file", "")), "") for params in requests.values()) + + def graph_language(path: str) -> str: + language = self.path_languages.get(path, "") + return "javascript" if language == "typescript" else language + + languages = dict.fromkeys(graph_language(str(params.get("file", ""))) for params in requests.values()) for language in languages: graph = self.graphs.get(language) if graph is None: continue if self.budget is not None and time.monotonic() >= self.budget.deadline_monotonic: break - selected = {rid: params for rid, params in requests.items() if self.path_languages.get(str(params.get("file", ""))) == language} + selected = {rid: params for rid, params in requests.items() if graph_language(str(params.get("file", ""))) == language} if not selected: continue batch = getattr(graph, method, None) or getattr(graph, "run_batch", None) @@ -133,7 +138,12 @@ class PartitionGraph: # Carried, not normalized: it names request ids and costs, never a path. answers.setdefault(TIMING, []).extend(rows) elif rid == "__census__" and isinstance(rows, list): - expected = set(next(p.paths for p in self.partitions if p.language == language)) + expected = { + path + for p in self.partitions + if p.language == language or (language == "javascript" and p.language == "typescript") + for path in p.paths + } normalized = self._normalize(rows) names = [r.get("file_names") for r in normalized if isinstance(r, Mapping)] named = bool(names) and all(isinstance(n, list) and all(isinstance(p, str) for p in n) for n in names) @@ -234,10 +244,17 @@ def build_partitions( graph.path_languages[relative] = languages[0] elif languages: boundaries.add("ambiguous_frontend_path") - if len(paths) > 1: + # JavaScript and TypeScript share one frontend input tree and graph. Keep + # original per-file languages for scoped coverage and property limitations. + groups: dict[str, list[str]] = {} + for language in paths: + key = "javascript" if language == "typescript" else language + groups.setdefault(key, []).append(language) + if len(groups) > 1: boundaries.add("cross_partition_semantics_unresolved") records: list[PartitionCoverage] = [] - for language, sources in sorted(paths.items()): + for language, members in sorted(groups.items()): + sources = [source for member in members for source in paths[member]] relative_paths = tuple(sorted(p.relative_to(root).as_posix() for p in sources)) frontend = FRONTENDS.get(language) if frontend is None: @@ -247,7 +264,7 @@ def build_partitions( continue scratch = Path(tempfile.mkdtemp(prefix="ousast-input-")) graph.roots[language] = scratch - size = 0 + sizes: dict[str, int] = {} complete = True for source in sources: if budget is not None and time.monotonic() >= budget.deadline_monotonic: @@ -268,25 +285,31 @@ def build_partitions( complete = False break output.write(chunk) - size += len(chunk) + relative = source.relative_to(root).as_posix() + sizes[relative] = sizes.get(relative, 0) + len(chunk) if not complete: break built = build(scratch, frontend) if complete else None if built is not None: graph.graphs[language] = built graph.unparsed += tuple(graph._normalize(list(getattr(built, "unparsed", ())))) - record_limits: tuple[str, ...] = ("runtime_unspecified",) if language in {"javascript", "typescript"} else () - if language == "typescript": - record_limits += ("typescript_property_support_unvalidated",) - boundaries.add("typescript_property_support_unvalidated") - # About TypeScript answers. YesWiki is a PHP application that ships a few `.ts` files, and this - # boundary made all 1,128 of its PHP questions unresolved. - owned_languages.setdefault("typescript_property_support_unvalidated", set()).add("typescript") - if language in {"c", "cpp"}: - record_limits += ("c_bounds_arithmetic_unmodeled",) - records.append( - PartitionCoverage(language, frontend, relative_paths, size, "built" if built is not None else "build_failed", record_limits) - ) + for language in sorted(members): + relative_paths = tuple(sorted(p.relative_to(root).as_posix() for p in paths[language])) + source_size = sum(sizes.get(p, 0) for p in relative_paths) + record_limits: tuple[str, ...] = ("runtime_unspecified",) if language in {"javascript", "typescript"} else () + if language == "typescript": + record_limits += ("typescript_property_support_unvalidated",) + boundaries.add("typescript_property_support_unvalidated") + # About TypeScript answers. YesWiki is a PHP application that ships a few `.ts` files, and this + # boundary made all 1,128 of its PHP questions unresolved. + owned_languages.setdefault("typescript_property_support_unvalidated", set()).add("typescript") + if language in {"c", "cpp"}: + record_limits += ("c_bounds_arithmetic_unmodeled",) + records.append( + PartitionCoverage( + language, frontend, relative_paths, source_size, "built" if built is not None else "build_failed", record_limits + ) + ) from dataclasses import replace graph.partitions = tuple( diff --git a/src/openultrasast/model/regions.py b/src/openultrasast/model/regions.py index 56a698c..4940ec0 100644 --- a/src/openultrasast/model/regions.py +++ b/src/openultrasast/model/regions.py @@ -24,6 +24,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from pathlib import Path +from ..semantic.facts import TAINT_LANGUAGE_ALIAS from .contracts import AffectedRelationship, ChangeContext, ExecutionBudget, QuestionIdentity from .specs import config_specs, dominance_specs, taint_specs @@ -41,7 +42,7 @@ _ACCESS_RANK: dict[str, float] = { # Languages whose facts the model layer carries. Anything else yields no region rather than an empty scan # that looks like a clean bill of health. _LANGUAGES = {"python", "javascript", "typescript", "java", "c", "c_cpp", "php"} -_NORMALISE = {"typescript": "javascript", "c_cpp": "c"} +_NORMALISE = {**TAINT_LANGUAGE_ALIAS, "c_cpp": "c"} # What a CPG calls a file's top-level code. A module body is a method like any other to the engine, and # naming it here is what lets a region ask about the code that is not inside any function. diff --git a/src/openultrasast/model/specs.py b/src/openultrasast/model/specs.py index 841b3df..7d58c71 100644 --- a/src/openultrasast/model/specs.py +++ b/src/openultrasast/model/specs.py @@ -21,7 +21,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass, field -from ..semantic.facts import SemanticFacts, load_facts +from ..semantic.facts import TAINT_LANGUAGE_ALIAS, SemanticFacts, load_facts from ..semantic.obligations.facts import ObligationFacts, load_obligation_facts from .taxonomy import FamilyTaxonomy, load_families @@ -245,7 +245,10 @@ def taint_specs( Sources and sanitizers are language-wide: a family does not get its own notion of what untrusted input is, and a sanitizer that breaks one flow breaks it whatever the sink's family. """ - scoped = (facts if facts is not None else load_facts()).for_language(language) + facts = facts if facts is not None else load_facts() + if language in TAINT_LANGUAGE_ALIAS: + language = facts.taint_language(language) + scoped = facts.for_language(language) families = taxonomy if taxonomy is not None else load_families() sources = tuple(sorted({pattern for fact in scoped.sources for pattern in fact.patterns})) diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index 8595c28..ec29d62 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -39,7 +39,7 @@ from .preprocess import FileTarget, preprocess_repository from .rank import rank_targets from .ruleset import PatternRule from .semantic import OverlayRecord, adjudicate -from .semantic.facts import FactLoadError, load_facts +from .semantic.facts import TAINT_LANGUAGE_ALIAS, FactLoadError, load_facts from .semantic.functions import RESERVED_FUNCTION_LABELS, function_at, named_function_ranges, spans_named from .semantic.obligations.facts import OPERATION_KINDS @@ -1071,7 +1071,7 @@ def _sink_ids_by_rule() -> dict[tuple[str, str], frozenset[str]]: def _fact_language(language: str) -> str: - return {"c_cpp": "c", "cpp": "c", "typescript": "javascript"}.get(language, language) + return {"c_cpp": "c", "cpp": "c", **TAINT_LANGUAGE_ALIAS}.get(language, language) def _in_named_function(record: OverlayRecord, function: str, ranges: FunctionRanges | None) -> bool: diff --git a/src/openultrasast/plane/tasks/alerts.py b/src/openultrasast/plane/tasks/alerts.py index 5c31044..3a64bfd 100644 --- a/src/openultrasast/plane/tasks/alerts.py +++ b/src/openultrasast/plane/tasks/alerts.py @@ -44,6 +44,7 @@ from ...mapping import analyze_entry_points, attach_reachability_hints from ...preprocess import detect_language, preprocess_repository from ...rank import rank_targets from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset +from ...semantic.facts import TAINT_LANGUAGE_ALIAS from .repo_facts import DECLARATION, GLOBAL, enclosing ROLES = ("vulnerable", "fixed") @@ -64,8 +65,9 @@ def quick_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]: def engine_languages(directory: Path = DEFAULT_RULESET_DIR) -> frozenset[str]: - """The languages the deterministic engine has a semantic model for: one ``semantic/.toml`` each.""" - return frozenset(path.stem for path in (directory / "semantic").glob("*.toml")) + """Languages with a semantic fact table, including aliases of available tables.""" + languages = {path.stem for path in (directory / "semantic").glob("*.toml")} + return frozenset(languages | {alias for alias, target in TAINT_LANGUAGE_ALIAS.items() if target in languages}) def covers(covered: Iterable[str], language: str) -> bool: diff --git a/src/openultrasast/preprocess.py b/src/openultrasast/preprocess.py index 8e7cb39..dcba5d8 100644 --- a/src/openultrasast/preprocess.py +++ b/src/openultrasast/preprocess.py @@ -22,6 +22,8 @@ LANGUAGE_BY_EXTENSION = { ".cjs": "javascript", ".ts": "typescript", ".tsx": "typescript", + ".mts": "typescript", + ".cts": "typescript", ".go": "go", ".rs": "rust", ".java": "java", diff --git a/src/openultrasast/semantic/facts.py b/src/openultrasast/semantic/facts.py index f7117c4..08c9b9e 100644 --- a/src/openultrasast/semantic/facts.py +++ b/src/openultrasast/semantic/facts.py @@ -15,12 +15,14 @@ from ..ruleset.frameworks import Priors, kept, normalize_priors, read_tag, tag_o DEFAULT_FACTS_DIR = Path(__file__).resolve().parents[1] / "ruleset" / "semantic" +TAINT_LANGUAGE_ALIAS = {"typescript": "javascript"} + _LANGUAGE_ALIASES = { "c": "c", "cpp": "c", "python": "python", "javascript": "javascript", - "typescript": "javascript", + **TAINT_LANGUAGE_ALIAS, "java": "java", } @@ -145,8 +147,18 @@ class SemanticFacts: dispatches: tuple[DispatchFact, ...] = () layouts: tuple[LayoutFact, ...] = () + def taint_language(self, language: str) -> str: + """Prefer native facts when supplied; otherwise use the shared language aliases.""" + if language in TAINT_LANGUAGE_ALIAS and any( + item.language == language + for items in (self.sources, self.sinks, self.sanitizers, self.dispatches, self.layouts) + for item in items + ): + return language + return _LANGUAGE_ALIASES.get(language, language) + def for_language(self, language: str) -> SemanticFacts: - key = _LANGUAGE_ALIASES.get(language, language) + key = self.taint_language(language) return SemanticFacts( version=self.version, sources=tuple(item for item in self.sources if item.language == key), diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 3778b32..29984ba 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -1368,3 +1368,33 @@ def test_requests_held_for_arbitration_are_reported_as_unasked() -> None: merged = scan._batched(once, "taint", {rid: {} for rid in weights}, weights=weights, within=lambda: next(budget, False), unasked=held) assert merged is not None and len(calls) == 1 assert sorted(held + calls[0]) == sorted(weights), (held, calls) + + +@pytest.mark.parametrize("extension", [".ts", ".tsx", ".mts", ".cts"]) +def test_typescript_uses_javascript_frontend(tmp_path, monkeypatch, extension): + from openultrasast.cpg.backend import JoernBackend + + (tmp_path / "tests").mkdir() + (tmp_path / f"tests/probe{extension}").write_bytes(b"function first_party_test(x) { return x; }") + from openultrasast.preprocess import detect_language + + assert detect_language(Path(f"probe{extension}")) == "typescript" + commands = [] + + def runner(command, **kwargs): + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + commands.append(command) + assert Path(command[0]).name == "jssrc2cpg" + Path(command[command.index("-o") + 1]).write_bytes(b"graph") + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}") + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + result = JoernBackend(runner=runner).build(tmp_path, language="typescript") + try: + assert result is not None + assert len(commands) == 1 + finally: + if result is not None: + result.cleanup() diff --git a/tests/test_learn_build_features.py b/tests/test_learn_build_features.py index dd3fb7e..4b2c61a 100644 --- a/tests/test_learn_build_features.py +++ b/tests/test_learn_build_features.py @@ -87,8 +87,8 @@ def test_feature_typescript_repository_witness_reaches_static_record(tmp_path: P assert parts["engine"].state == "ran" assert parts["engine"].values["eng.witness_steps_min"] == 5 assert parts["engine"].values["eng.findings"] == 1 - assert builder.recorded_engine("typescript", [], None, True).state == "none" - assert builder.recorded_engine("typescript", [], result, False).state == "none" + assert builder.recorded_engine("typescript", [], None, True).state == "failed" + assert builder.recorded_engine("typescript", [], result, False).state == "ran" assert builder.recorded_engine("typescript", [], result, True).values["eng.findings"] == 0 diff --git a/tests/test_model_partitions.py b/tests/test_model_partitions.py index e8e600b..fa0010b 100644 --- a/tests/test_model_partitions.py +++ b/tests/test_model_partitions.py @@ -344,3 +344,37 @@ def test_unnamed_sharded_census_remains_explicitly_incomplete(tmp_path): assert "partition_file_census_unavailable" in graph.execution_diagnostics() finally: graph.cleanup() + + +def test_mixed_javascript_typescript_pin_builds_and_queries_one_graph(tmp_path): + from openultrasast.model.partitions import build_partitions + + sources = {"app.js": "function run(x) { return x; }", "handler.ts": "const x: number = 1;"} + for path, body in sources.items(): + (tmp_path / path).write_text(body) + builds, queries = [], [] + + class Graph: + def run_batch(self, kind, requests): + queries.append(requests) + return {**{key: [] for key in requests}, "__census__": [{"file_names": list(sources)}]} + + def build(root, language): + assert {p.name: p.read_text() for p in root.iterdir()} == sources + builds.append(language) + return Graph() + + graph = build_partitions(tmp_path, build, None) + try: + assert builds == ["javascript"] + assert {p.language for p in graph.partitions} == {"javascript", "typescript"} + assert sum(p.source_bytes for p in graph.partitions) == sum(len(body.encode()) for body in sources.values()) + assert "cross_partition_semantics_unresolved" not in graph.boundaries + assert graph.boundary_languages["typescript_property_support_unvalidated"] == ("typescript",) + requests = {path: {"file": path} for path in sources} + answers = graph.run_batch("taint", requests) + assert all(answers[path] == [] for path in sources) + assert queries == [requests] + assert not graph.execution_diagnostics() + finally: + graph.cleanup() diff --git a/tests/test_model_specs.py b/tests/test_model_specs.py index 4134120..6388b56 100644 --- a/tests/test_model_specs.py +++ b/tests/test_model_specs.py @@ -322,3 +322,26 @@ def test_a_sanitizer_that_does_not_stop_sql_injection_is_not_listed() -> None: assert "esc_html" not in sanitizers assert "$wpdb->prepare" in sanitizers, "the parameterized form does stop it" assert "esc_sql" in sanitizers + + +def test_typescript_taint_specs_share_javascript_facts(): + from openultrasast.model.specs import taint_specs + + javascript = taint_specs(language="javascript") + assert javascript + assert taint_specs(language="typescript") == javascript + + +def test_typescript_native_facts_take_precedence_over_alias(): + from dataclasses import replace + + from openultrasast.model.specs import taint_specs + from openultrasast.semantic.facts import load_facts + + facts = load_facts() + native = replace(next(sink for sink in facts.sinks if sink.language == "javascript"), language="typescript", calls=("tsOnly",)) + facts = replace(facts, sinks=(*facts.sinks, native)) + specs = taint_specs(language="typescript", facts=facts) + assert specs + assert {spec.language for spec in specs.values()} == {"typescript"} + assert {sink for spec in specs.values() for sink in spec.sinks} == {"tsOnly"} diff --git a/tests/test_plane_loop.py b/tests/test_plane_loop.py index dda741c..965c05d 100644 --- a/tests/test_plane_loop.py +++ b/tests/test_plane_loop.py @@ -311,3 +311,15 @@ def test_the_loop_runs_as_one_plane_run_and_the_gate_rejects_a_demotion_that_cos assert set(features) == {"lmdeploy/vl/media/connection.py::_load_http_url"}, "no fixed-pin alert became a candidate" assert all(r["pin"] == VULNERABLE and "in_fix_range" not in json.dumps(r["x"]) for r in features.values()) assert fixed_alert.row["in_fix_range"] is True and fixed_alert.row["function"] == "load" + + +def test_engine_languages_include_aliases_only_for_available_tables(tmp_path): + from openultrasast.plane.tasks.alerts import engine_languages + + assert "typescript" in engine_languages() + semantic = tmp_path / "semantic" + semantic.mkdir() + (semantic / "python.toml").write_text("") + assert engine_languages(tmp_path) == {"python"} + (semantic / "javascript.toml").write_text("") + assert engine_languages(tmp_path) == {"python", "javascript", "typescript"}