mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learning-harness): TypeScript and TSX rows parse under the semantic extra
Task 1.4. The optional extra gains the TypeScript grammar wheel, which ships two grammars: a file with JSX needs the JSX-aware one and plain TypeScript needs the other, so the parser picks by file suffix while the parsed file keeps the language it was given. The walker already treated TypeScript as JavaScript, so nothing else changes. Eighteen agent-vfc rows and every TypeScript absence row were counted as unsupported before this and could not be scored at all. They now parse on both sides. The core install stays zero-dependency and the extra-free suite still skips these tests. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
6c205b728f
commit
6b58c81ba0
@@ -26,6 +26,7 @@ semantic = [
|
||||
"tree-sitter>=0.23,<0.27",
|
||||
"tree-sitter-python>=0.23",
|
||||
"tree-sitter-javascript>=0.23",
|
||||
"tree-sitter-typescript>=0.23",
|
||||
"tree-sitter-c>=0.23",
|
||||
"tree-sitter-cpp>=0.23",
|
||||
"tree-sitter-java>=0.23",
|
||||
|
||||
@@ -42,6 +42,7 @@ _KIND = {
|
||||
}
|
||||
_KIND["cpp"] = _KIND["c"]
|
||||
_KIND["typescript"] = _KIND["javascript"]
|
||||
_KIND["tsx"] = _KIND["javascript"]
|
||||
|
||||
_CONST = frozenset(
|
||||
{
|
||||
@@ -64,7 +65,7 @@ _SEQUENCE = frozenset({"list", "tuple", "array", "array_creation_expression", "i
|
||||
def parse_with_cst(path: str, text: str, language: str) -> FileIR | None:
|
||||
from .extra import grammar_for
|
||||
|
||||
grammar = grammar_for(language)
|
||||
grammar = grammar_for(_grammar_language(language, path))
|
||||
if grammar is None:
|
||||
return None
|
||||
try:
|
||||
@@ -77,6 +78,11 @@ def parse_with_cst(path: str, text: str, language: str) -> FileIR | None:
|
||||
return file_ir_from_tree(path=path, text=text, language=language, tree=tree)
|
||||
|
||||
|
||||
def _grammar_language(language: str, path: str) -> str:
|
||||
"""TypeScript ships two grammars; a `.tsx` file needs the JSX-aware one. The FileIR keeps the language it was given."""
|
||||
return "tsx" if language == "typescript" and path.endswith(".tsx") else language
|
||||
|
||||
|
||||
def file_ir_from_tree(*, path: str, text: str, language: str, tree: Any) -> FileIR:
|
||||
del text
|
||||
root = tree.root_node
|
||||
|
||||
@@ -7,12 +7,16 @@ import importlib.util
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
# language -> (wheel module, the module attribute that builds the grammar). TypeScript ships two grammars;
|
||||
# `.tsx` needs the JSX-aware one and plain `.ts` needs the other, so the caller asks for the one it needs.
|
||||
_GRAMMAR_MODULES = {
|
||||
"python": "tree_sitter_python",
|
||||
"javascript": "tree_sitter_javascript",
|
||||
"c": "tree_sitter_c",
|
||||
"cpp": "tree_sitter_cpp",
|
||||
"java": "tree_sitter_java",
|
||||
"python": ("tree_sitter_python", "language"),
|
||||
"javascript": ("tree_sitter_javascript", "language"),
|
||||
"typescript": ("tree_sitter_typescript", "language_typescript"),
|
||||
"tsx": ("tree_sitter_typescript", "language_tsx"),
|
||||
"c": ("tree_sitter_c", "language"),
|
||||
"cpp": ("tree_sitter_cpp", "language"),
|
||||
"java": ("tree_sitter_java", "language"),
|
||||
}
|
||||
_PROBE_ENV = "OPENULTRASAST_TREE_SITTER_PROBE"
|
||||
_PROBE_OFF = frozenset({"0", "off", "false", "unavailable", "no"})
|
||||
@@ -26,7 +30,7 @@ def has_semantic_extra() -> bool:
|
||||
return False
|
||||
if importlib.util.find_spec("tree_sitter") is None:
|
||||
return False
|
||||
return any(importlib.util.find_spec(module) is not None for module in _GRAMMAR_MODULES.values())
|
||||
return any(importlib.util.find_spec(module) is not None for module, _ in _GRAMMAR_MODULES.values())
|
||||
|
||||
|
||||
def grammar_for(language: str) -> Any | None:
|
||||
@@ -34,15 +38,16 @@ def grammar_for(language: str) -> Any | None:
|
||||
flag = os.environ.get(_PROBE_ENV, "").strip().lower()
|
||||
if flag in _PROBE_OFF:
|
||||
return None
|
||||
module_name = _GRAMMAR_MODULES.get(language)
|
||||
if module_name is None or importlib.util.find_spec("tree_sitter") is None:
|
||||
entry = _GRAMMAR_MODULES.get(language)
|
||||
if entry is None or importlib.util.find_spec("tree_sitter") is None:
|
||||
return None
|
||||
module_name, attribute = entry
|
||||
if importlib.util.find_spec(module_name) is None:
|
||||
return None
|
||||
try:
|
||||
from tree_sitter import Language
|
||||
|
||||
module = importlib.import_module(module_name)
|
||||
return Language(module.language())
|
||||
return Language(getattr(module, attribute)())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""learning-harness task 1.4: TypeScript and TSX rows parse under the semantic extra."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored
|
||||
from openultrasast.semantic.extra import has_semantic_extra
|
||||
from openultrasast.semantic.ir import parse_file
|
||||
|
||||
pytestmark = pytest.mark.semantic
|
||||
|
||||
TS = (
|
||||
"export async function GET(request: NextRequest): Promise<Response> {\n"
|
||||
" const id = request.nextUrl.searchParams.get('id');\n"
|
||||
" return NextResponse.json(await db.from('claims').select().eq('id', id));\n}\n"
|
||||
)
|
||||
TSX = (
|
||||
"export function Panel({ token }: { token: string }) {\n"
|
||||
" const [value, setValue] = useState('');\n"
|
||||
' return <div className="p">{value}{token}</div>;\n}\n'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_semantic_extra(), reason="requires the semantic extra")
|
||||
def test_typescript_and_tsx_parse_instead_of_reporting_an_unsupported_language() -> None:
|
||||
ir = parse_file("app/api/route.ts", TS, "typescript")
|
||||
assert ir.parse_ok and ir.reason is None and ir.engine == "tree-sitter"
|
||||
assert "GET" in {function.name for function in ir.functions}
|
||||
tsx = parse_file("components/panel.tsx", TSX, "typescript")
|
||||
assert tsx.parse_ok and tsx.reason is None # the TSX grammar, chosen by the file suffix
|
||||
assert "Panel" in {function.name for function in tsx.functions}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_semantic_extra(), reason="requires the semantic extra")
|
||||
def test_every_vendored_typescript_pair_parses_on_both_sides() -> None:
|
||||
cases = [case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if case.language == "typescript"]
|
||||
assert len(cases) >= 18
|
||||
unsupported: list[str] = []
|
||||
failed: list[str] = []
|
||||
for case in cases:
|
||||
for side, path in (("vuln", case.vuln_file), ("fixed", case.fixed_file)):
|
||||
ir = parse_file(str(path), path.read_text(errors="ignore"), "typescript")
|
||||
if ir.reason == "language_unsupported":
|
||||
unsupported.append(f"{case.name}:{side}")
|
||||
elif not ir.parse_ok:
|
||||
failed.append(f"{case.name}:{side}")
|
||||
assert unsupported == []
|
||||
assert len(failed) <= len(cases) // 4, failed # a parse failure is a walker gap, not a missing grammar
|
||||
@@ -6,8 +6,8 @@ resolution-markers = [
|
||||
"python_full_version == '3.14.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.14.*' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'win32'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32'",
|
||||
"python_full_version < '3.13' and sys_platform != 'win32'",
|
||||
]
|
||||
|
||||
@@ -2152,6 +2152,7 @@ semantic = [
|
||||
{ name = "tree-sitter-java" },
|
||||
{ name = "tree-sitter-javascript" },
|
||||
{ name = "tree-sitter-python" },
|
||||
{ name = "tree-sitter-typescript" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
@@ -2172,6 +2173,7 @@ requires-dist = [
|
||||
{ name = "tree-sitter-java", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
{ name = "tree-sitter-javascript", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
{ name = "tree-sitter-python", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
{ name = "tree-sitter-typescript", marker = "extra == 'semantic'", specifier = ">=0.23" },
|
||||
]
|
||||
provides-extras = ["harnessx", "semantic"]
|
||||
|
||||
@@ -3469,6 +3471,21 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-typescript"
|
||||
version = "0.23.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typer"
|
||||
version = "0.25.1"
|
||||
|
||||
Reference in New Issue
Block a user