diff --git a/pyproject.toml b/pyproject.toml index 9cf9c28..cba7a91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ semantic = [ "tree-sitter>=0.23,<0.27", "tree-sitter-python>=0.23", "tree-sitter-javascript>=0.23", + "tree-sitter-typescript>=0.23", "tree-sitter-c>=0.23", "tree-sitter-cpp>=0.23", "tree-sitter-java>=0.23", diff --git a/src/openultrasast/semantic/cst.py b/src/openultrasast/semantic/cst.py index 04e64f9..0e754d7 100644 --- a/src/openultrasast/semantic/cst.py +++ b/src/openultrasast/semantic/cst.py @@ -42,6 +42,7 @@ _KIND = { } _KIND["cpp"] = _KIND["c"] _KIND["typescript"] = _KIND["javascript"] +_KIND["tsx"] = _KIND["javascript"] _CONST = frozenset( { @@ -64,7 +65,7 @@ _SEQUENCE = frozenset({"list", "tuple", "array", "array_creation_expression", "i def parse_with_cst(path: str, text: str, language: str) -> FileIR | None: from .extra import grammar_for - grammar = grammar_for(language) + grammar = grammar_for(_grammar_language(language, path)) if grammar is None: return None try: @@ -77,6 +78,11 @@ def parse_with_cst(path: str, text: str, language: str) -> FileIR | None: return file_ir_from_tree(path=path, text=text, language=language, tree=tree) +def _grammar_language(language: str, path: str) -> str: + """TypeScript ships two grammars; a `.tsx` file needs the JSX-aware one. The FileIR keeps the language it was given.""" + return "tsx" if language == "typescript" and path.endswith(".tsx") else language + + def file_ir_from_tree(*, path: str, text: str, language: str, tree: Any) -> FileIR: del text root = tree.root_node diff --git a/src/openultrasast/semantic/extra.py b/src/openultrasast/semantic/extra.py index 30a3f12..62df964 100644 --- a/src/openultrasast/semantic/extra.py +++ b/src/openultrasast/semantic/extra.py @@ -7,12 +7,16 @@ import importlib.util import os from typing import Any +# language -> (wheel module, the module attribute that builds the grammar). TypeScript ships two grammars; +# `.tsx` needs the JSX-aware one and plain `.ts` needs the other, so the caller asks for the one it needs. _GRAMMAR_MODULES = { - "python": "tree_sitter_python", - "javascript": "tree_sitter_javascript", - "c": "tree_sitter_c", - "cpp": "tree_sitter_cpp", - "java": "tree_sitter_java", + "python": ("tree_sitter_python", "language"), + "javascript": ("tree_sitter_javascript", "language"), + "typescript": ("tree_sitter_typescript", "language_typescript"), + "tsx": ("tree_sitter_typescript", "language_tsx"), + "c": ("tree_sitter_c", "language"), + "cpp": ("tree_sitter_cpp", "language"), + "java": ("tree_sitter_java", "language"), } _PROBE_ENV = "OPENULTRASAST_TREE_SITTER_PROBE" _PROBE_OFF = frozenset({"0", "off", "false", "unavailable", "no"}) @@ -26,7 +30,7 @@ def has_semantic_extra() -> bool: return False if importlib.util.find_spec("tree_sitter") is None: return False - return any(importlib.util.find_spec(module) is not None for module in _GRAMMAR_MODULES.values()) + return any(importlib.util.find_spec(module) is not None for module, _ in _GRAMMAR_MODULES.values()) def grammar_for(language: str) -> Any | None: @@ -34,15 +38,16 @@ def grammar_for(language: str) -> Any | None: flag = os.environ.get(_PROBE_ENV, "").strip().lower() if flag in _PROBE_OFF: return None - module_name = _GRAMMAR_MODULES.get(language) - if module_name is None or importlib.util.find_spec("tree_sitter") is None: + entry = _GRAMMAR_MODULES.get(language) + if entry is None or importlib.util.find_spec("tree_sitter") is None: return None + module_name, attribute = entry if importlib.util.find_spec(module_name) is None: return None try: from tree_sitter import Language module = importlib.import_module(module_name) - return Language(module.language()) + return Language(getattr(module, attribute)()) except Exception: return None diff --git a/tests/test_typescript_grammar.py b/tests/test_typescript_grammar.py new file mode 100644 index 0000000..3153cb3 --- /dev/null +++ b/tests/test_typescript_grammar.py @@ -0,0 +1,49 @@ +"""learning-harness task 1.4: TypeScript and TSX rows parse under the semantic extra.""" + +from __future__ import annotations + +import pytest + +from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored +from openultrasast.semantic.extra import has_semantic_extra +from openultrasast.semantic.ir import parse_file + +pytestmark = pytest.mark.semantic + +TS = ( + "export async function GET(request: NextRequest): Promise {\n" + " const id = request.nextUrl.searchParams.get('id');\n" + " return NextResponse.json(await db.from('claims').select().eq('id', id));\n}\n" +) +TSX = ( + "export function Panel({ token }: { token: string }) {\n" + " const [value, setValue] = useState('');\n" + ' return
{value}{token}
;\n}\n' +) + + +@pytest.mark.skipif(not has_semantic_extra(), reason="requires the semantic extra") +def test_typescript_and_tsx_parse_instead_of_reporting_an_unsupported_language() -> None: + ir = parse_file("app/api/route.ts", TS, "typescript") + assert ir.parse_ok and ir.reason is None and ir.engine == "tree-sitter" + assert "GET" in {function.name for function in ir.functions} + tsx = parse_file("components/panel.tsx", TSX, "typescript") + assert tsx.parse_ok and tsx.reason is None # the TSX grammar, chosen by the file suffix + assert "Panel" in {function.name for function in tsx.functions} + + +@pytest.mark.skipif(not has_semantic_extra(), reason="requires the semantic extra") +def test_every_vendored_typescript_pair_parses_on_both_sides() -> None: + cases = [case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if case.language == "typescript"] + assert len(cases) >= 18 + unsupported: list[str] = [] + failed: list[str] = [] + for case in cases: + for side, path in (("vuln", case.vuln_file), ("fixed", case.fixed_file)): + ir = parse_file(str(path), path.read_text(errors="ignore"), "typescript") + if ir.reason == "language_unsupported": + unsupported.append(f"{case.name}:{side}") + elif not ir.parse_ok: + failed.append(f"{case.name}:{side}") + assert unsupported == [] + assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar diff --git a/uv.lock b/uv.lock index fb3a8b8..5a57a9e 100644 --- a/uv.lock +++ b/uv.lock @@ -6,8 +6,8 @@ resolution-markers = [ "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version < '3.13' and sys_platform != 'win32'", ] @@ -2152,6 +2152,7 @@ semantic = [ { name = "tree-sitter-java" }, { name = "tree-sitter-javascript" }, { name = "tree-sitter-python" }, + { name = "tree-sitter-typescript" }, ] [package.dev-dependencies] @@ -2172,6 +2173,7 @@ requires-dist = [ { name = "tree-sitter-java", marker = "extra == 'semantic'", specifier = ">=0.23" }, { name = "tree-sitter-javascript", marker = "extra == 'semantic'", specifier = ">=0.23" }, { name = "tree-sitter-python", marker = "extra == 'semantic'", specifier = ">=0.23" }, + { name = "tree-sitter-typescript", marker = "extra == 'semantic'", specifier = ">=0.23" }, ] provides-extras = ["harnessx", "semantic"] @@ -3469,6 +3471,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" }, ] +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" }, + { url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" }, +] + [[package]] name = "typer" version = "0.25.1"