diff --git a/pyproject.toml b/pyproject.toml index c039b0c..c9ef92c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ docs = [ s3 = [ "boto3>=1.35,<2", ] -# Structured parse for overlay (JS/C/C++/Java/Python). Core install stays empty. +# Structured parse for overlay (JS/C/C++/Java/PHP/Python). Core install stays empty. # Wheels are MIT; import is lazy via openultrasast.semantic.extra. semantic = [ "tree-sitter>=0.23,<0.27", @@ -41,6 +41,7 @@ semantic = [ "tree-sitter-c>=0.23", "tree-sitter-cpp>=0.23", "tree-sitter-java>=0.23", + "tree-sitter-php>=0.23", ] [project.scripts] diff --git a/src/openultrasast/hunt/features.py b/src/openultrasast/hunt/features.py index 665d11c..e669201 100644 --- a/src/openultrasast/hunt/features.py +++ b/src/openultrasast/hunt/features.py @@ -47,6 +47,7 @@ def _engineer(row: Mapping[str, Any]) -> dict[str, Any]: callers = row.get("feature.source.callers") roles = row.get("feature.semantic.roles") or None def_use = row.get("feature.semantic.def_use") or None + # None means unchecked even when roles and def_use succeeded; [] is checked. obligations = row.get("feature.semantic.obligations") function = row.get("feature.source.function") out: dict[str, Any] = {key: row.get(key) for key in KEYS} diff --git a/src/openultrasast/hunt/producers.py b/src/openultrasast/hunt/producers.py index 4a0f037..537a3a1 100644 --- a/src/openultrasast/hunt/producers.py +++ b/src/openultrasast/hunt/producers.py @@ -211,7 +211,7 @@ def _source(root: Path, change: Change, path: str, line: int, ctx: Context) -> d ctx.read(path) if path in ctx.range_errors: raise _Absent(ctx.range_errors[path]) - facts = source.facts_for(root, path, line, entry_points=ctx.entry_points) + facts = source.facts_for(root, path, line, entry_points=ctx.entry_points, files=ctx.product_files, texts=ctx.texts) # Select the occurrence at this line when several methods share a name. spans = [r for r in ctx.ranges(path) if r.name == facts.function and r.start <= line <= r.end] span = min(spans, key=lambda r: r.end - r.start) if spans else None diff --git a/src/openultrasast/ruleset/obligations/java.toml b/src/openultrasast/ruleset/obligations/java.toml new file mode 100644 index 0000000..6a97af2 --- /dev/null +++ b/src/openultrasast/ruleset/obligations/java.toml @@ -0,0 +1,64 @@ +# Closed authorization facts for Jenkins/Stapler, Spring Security and Servlet code. +# Checked Git HEAD source objects in /home/mc/ousast-results/valid/train/hist/: +# jenkinsci__analysis-core-plugin: DefaultGraphConfigurationView.checkPermission, +# AbstractAnnotation.hasPermission and GlobalSettings.save; android-signing-plugin: +# SignApksBuilder.lookupCredentials; ansible-plugin: AnsibleAdHocCommandBuilder.findCredentialById; +# blueocean-plugin: GitScm.getItemByFullName; cloud-stats-plugin: CloudStatistics.getConfigFile; +# job-import-plugin: JobImportAction.updateByXml. These are real calls, not prose matches. +# checkAnyPermission is documented by Jenkins AccessControlled, although absent from these HEADs: +# https://javadoc.jenkins.io/hudson/security/AccessControlled.html +# checkReadPermission was not found in the clones or API references and is not guessed here. +# Spring and Servlet are already named in ruleset/semantic/java.toml. Their identity/check APIs: +# https://docs.spring.io/spring-security/site/docs/current/api/org/springframework/security/acls/model/Acl.html +# https://docs.spring.io/spring-security/site/docs/5.4.x/reference/html5/ (Servlet integration) +# Calls use trailing method names. Boolean predicates are ownership_check witnesses, requiring +# an exit before the operation under OrderDominance; only throwing checks are path_guard facts. +version = "1" +language = "java" + +[[operation]] +id = "jenkins-credentials-read" +kind = "protected_read" +calls = ["lookupCredentials", "findCredentialById"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +# A Stapler doXxx handler is not itself a protected operation; its item lookup is. +id = "jenkins-item-read" +kind = "protected_read" +calls = ["getItemByFullName", "getConfigFile"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +id = "jenkins-item-write" +kind = "protected_write" +calls = ["updateByXml", "save"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +# JDBC access already named in semantic/java.toml; authorization is separate from SQL escaping. +id = "sql-access" +kind = "protected_read" +calls = ["executeQuery", "execute", "prepareCall"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[discharger]] +id = "jenkins-permission-guard" +kind = "path_guard" +calls = ["checkPermission", "checkAnyPermission"] + +[[discharger]] +id = "permission-predicate" +kind = "ownership_check" +calls = ["hasPermission", "isGranted", "isUserInRole"] + +[[discharger]] +# Reading the security context alone is not an authorization guard. +id = "identity-from-context" +kind = "identity_constraint" +identity_sources = ["SecurityContextHolder.getContext().getAuthentication()", "Jenkins.getAuthentication()", "Jenkins.getAuthentication2()", "getUserPrincipal()"] +request_sources = ["getParameter(", "getHeader(", "getCookies(", "getQueryString("] diff --git a/src/openultrasast/ruleset/obligations/php.toml b/src/openultrasast/ruleset/obligations/php.toml new file mode 100644 index 0000000..8e12368 --- /dev/null +++ b/src/openultrasast/ruleset/obligations/php.toml @@ -0,0 +1,93 @@ +# Closed authorization facts for Laravel, WordPress and Symfony. +# Checked shopperlabs__shopper Git HEAD source objects in /home/mc/ousast-results/valid/train/hist/: +# CollectionProducts.authorize, SetupGuide.can, AssetController.abort_unless, +# OrderCustomer.find, SaveProductVariantsAction.findOrFail, InitialQuantityInventory.first, +# UpdateProductAction.update, DetachAttributesToProductAction.delete, UserCommand.save, +# PermissionsTableSeeder.create, and Account/Devices.php's user_id constraint from auth()->user(). +# Gate predicates, abort_if and destroy are framework APIs, not observed Shopper HEAD calls: +# https://laravel.com/framework/docs/10.x/authorization (Gate and policy dispatch) +# https://api.laravel.com/docs/10.x/%5BGlobal_Namespace%5D.html (abort helpers) +# https://api.laravel.com/docs/12.x/Illuminate/Database/Eloquent/Model.html +# WordPress is already covered by semantic/php.toml (including the $wpdb calls below). +# Post, option, capability, login and nonce APIs were checked in the official function reference: +# https://developer.wordpress.org/reference/functions/get_post/ +# https://developer.wordpress.org/reference/functions/wp_update_post/ +# https://developer.wordpress.org/reference/functions/update_option/ +# https://developer.wordpress.org/reference/functions/current_user_can/ +# https://developer.wordpress.org/reference/functions/is_user_logged_in/ +# https://developer.wordpress.org/reference/functions/check_admin_referer/ +# https://developer.wordpress.org/reference/functions/wp_verify_nonce/ +# Symfony's controller checks: https://symfony.com/doc/current/security.html +# Calls use trailing names: Gate::authorize and $this->authorize both become authorize; +# variables retain their $ sigil, consistent with the CST and existing PHP source facts. +# Common Eloquent/WordPress names are annotations, not receiver type resolution or proof. +version = "1" +language = "php" + +[[operation]] +id = "eloquent-read" +kind = "protected_read" +calls = ["find", "findOrFail", "first", "get"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +id = "eloquent-write" +kind = "protected_write" +calls = ["update", "delete", "destroy", "save", "create"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +id = "wordpress-read" +kind = "protected_read" +calls = ["get_post", "get_var", "get_results", "get_row", "get_col", "query"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +id = "wordpress-write" +kind = "protected_write" +calls = ["wp_update_post", "replace"] +requires = ["path_guard", "ownership_check", "identity_constraint"] +sensitivity = "high" + +[[operation]] +id = "wordpress-option-write" +kind = "privileged_action" +calls = ["update_option"] +requires = ["path_guard", "ownership_check"] +sensitivity = "high" + +[[discharger]] +# Laravel policy enforcement and Symfony's throwing controller guard. +id = "authorization-guard" +kind = "path_guard" +calls = ["authorize", "denyAccessUnlessGranted"] + +[[discharger]] +# These return booleans: a bare call is not a guard. OrderDominance requires an exit. +id = "permission-predicate" +kind = "ownership_check" +calls = ["can", "cannot", "allows", "denies", "current_user_can", "isGranted"] + +[[discharger]] +# These throw conditionally; as with other guards, this annotation cannot establish +# that the condition is the right permission check. It is not a proof of authorization. +id = "conditional-abort" +kind = "path_guard" +calls = ["abort_if", "abort_unless"] + +[[discharger]] +# Nonces validate intent, not capabilities; login alone does not grant a permission. +# None of the protected operations above accept validated_input as authorization. +id = "wordpress-request-validation" +kind = "validated_input" +calls = ["check_admin_referer", "wp_verify_nonce", "is_user_logged_in"] + +[[discharger]] +id = "identity-from-context" +kind = "identity_constraint" +identity_sources = ["auth()->user()", "shopper()->auth()->user()"] +constraint_params = ["user_id"] +request_sources = ["$_GET", "$_POST", "$_REQUEST", "$_COOKIE", "$_FILES"] diff --git a/src/openultrasast/semantic/cst.py b/src/openultrasast/semantic/cst.py index b96b379..3c683f7 100644 --- a/src/openultrasast/semantic/cst.py +++ b/src/openultrasast/semantic/cst.py @@ -7,6 +7,16 @@ from typing import Any from .ir import Bind, CallSite, FileIR, FunctionIR _KIND = { + "php": { + "function": frozenset({"function_definition", "method_declaration", "anonymous_function", "arrow_function"}), + "assign": frozenset({"assignment_expression", "augmented_assignment_expression"}), + "call": frozenset( + {"function_call_expression", "member_call_expression", "scoped_call_expression", "nullsafe_member_call_expression"} + ), + "args": frozenset({"arguments"}), + "string": frozenset({"string", "encapsed_string"}), + "params": frozenset({"formal_parameters"}), + }, "python": { "function": frozenset({"function_definition"}), "assign": frozenset({"assignment", "augmented_assignment"}), @@ -170,6 +180,8 @@ def _bind(node: Any, kinds: dict[str, frozenset[str]]) -> Bind | None: def _call(node: Any, kinds: dict[str, frozenset[str]]) -> CallSite: args_node = next((child for child in getattr(node, "named_children", ()) or () if child.type in kinds["args"]), None) arg_nodes = list(getattr(args_node, "named_children", ()) or ()) if args_node is not None else [] + # PHP wraps positional and named argument values in an `argument` node. + arg_nodes = [arg.named_children[-1] if arg.type == "argument" and arg.named_children else arg for arg in arg_nodes] extra = len(arg_nodes) >= 2 and arg_nodes[1].type in _SEQUENCE return CallSite( order=int(node.end_byte), @@ -183,6 +195,10 @@ def _call(node: Any, kinds: dict[str, frozenset[str]]) -> CallSite: def _callee_name(node: Any, args_node: Any) -> str: + if node.type in _KIND["php"]["call"]: + callee = node.child_by_field_name("name") or node.child_by_field_name("function") + if callee is not None: + return _text(callee).rsplit("\\", 1)[-1] idents = [ _text(child) for child in getattr(node, "named_children", ()) or () @@ -207,7 +223,7 @@ def _call_name(node: Any, kinds: dict[str, frozenset[str]]) -> str | None: def _function_name(node: Any) -> str: for child in getattr(node, "named_children", ()) or (): - if child.type == "identifier": + if child.type in {"identifier", "name"}: return _text(child) if child.type == "function_declarator": for grandchild in getattr(child, "named_children", ()) or (): @@ -222,6 +238,10 @@ def _params(node: Any, kinds: dict[str, frozenset[str]]) -> tuple[str, ...]: return _identifiers(parameter) for child in _walk(node, skip=frozenset()): if child.type in kinds["params"]: + if any( + param.type in {"simple_parameter", "variadic_parameter", "property_promotion_parameter"} for param in child.named_children + ): + return tuple(_text(name) for param in child.named_children if (name := param.child_by_field_name("name")) is not None) return _identifiers(child) if child.type == "function_declarator": for grandchild in getattr(child, "named_children", ()) or (): @@ -231,7 +251,7 @@ def _params(node: Any, kinds: dict[str, frozenset[str]]) -> tuple[str, ...]: def _left_name(node: Any) -> str: - if node.type in {"identifier", "property_identifier"}: + if node.type in {"identifier", "property_identifier", "variable_name"}: return _text(node) idents = _identifiers(node) return idents[0] if idents else "" @@ -240,12 +260,16 @@ def _left_name(node: Any) -> str: def _identifiers(node: Any) -> tuple[str, ...]: names: list[str] = [] for child in _walk(node): - if child.type == "identifier": + # Keep PHP's sigil, matching source facts such as $_GET and $_POST. The + # nested `name` node is not another variable (nor are class/type names). + if child.type in {"identifier", "variable_name"}: names.append(_text(child)) return tuple(names) def _is_constant(node: Any, kinds: dict[str, frozenset[str]]) -> bool: + if node.type == "encapsed_string": + return all(child.type in {"string_content", "escape_sequence"} for child in node.named_children) if node.type in _CONST or node.type in kinds["string"]: return True if node.type not in {"binary_expression", "concatenated_string"}: diff --git a/src/openultrasast/semantic/extra.py b/src/openultrasast/semantic/extra.py index 62df964..08092ca 100644 --- a/src/openultrasast/semantic/extra.py +++ b/src/openultrasast/semantic/extra.py @@ -17,6 +17,7 @@ _GRAMMAR_MODULES = { "c": ("tree_sitter_c", "language"), "cpp": ("tree_sitter_cpp", "language"), "java": ("tree_sitter_java", "language"), + "php": ("tree_sitter_php", "language_php"), } _PROBE_ENV = "OPENULTRASAST_TREE_SITTER_PROBE" _PROBE_OFF = frozenset({"0", "off", "false", "unavailable", "no"}) diff --git a/tests/test_hunt_features.py b/tests/test_hunt_features.py index 24809cb..8f328b4 100644 --- a/tests/test_hunt_features.py +++ b/tests/test_hunt_features.py @@ -100,3 +100,14 @@ def test_screen_reports_the_strongest_feature_per_annotation(): assert out.loc["hunks", "strongest_feature"] == "diff.distance" and out.loc["hunks", "auc"] == 0.0 assert out.loc["callers", "strongest_feature"] == "source.caller_count" and out.loc["callers", "auc"] == 1.0 assert out.loc["roles", "auc"] == 0.5 # constant: no signal + + +def test_unchecked_obligations_are_missing_while_checked_empty_is_zero(): + frame = features.feature_frame([{**ROW, "feature.semantic.obligations": value} for value in (None, [])]) + for column in ("semantic.obligations", "semantic.obligations_undischarged"): + assert pd.isna(frame.iloc[0][column]) + assert frame.iloc[1][column] == 0 + for row in frame.to_dict("records"): + assert row["ok.semantic"] + assert row["semantic.sources"] == 1 + assert row["semantic.calls"] == 2 diff --git a/tests/test_hunt_producers.py b/tests/test_hunt_producers.py index 4b20d1f..8168668 100644 --- a/tests/test_hunt_producers.py +++ b/tests/test_hunt_producers.py @@ -198,3 +198,61 @@ def test_a_removed_guard_anchors_the_function_it_left(tmp_path): assert row["feature.diff.on_changed_line"] and row["feature.diff.distance"] == 0 first = next(r for r in rows if r["feature.source.function"] == "first") assert first["feature.diff.distance"] == 4 and not first["feature.diff.on_changed_line"] + + +@pytest.mark.parametrize( + ("path", "text", "source_name", "operation"), + [ + ( + "App.java", + 'class App {\n void run() {\n String id = request.getParameter("id");\n' + " checkPermission(ADMINISTER);\n getItemByFullName(id);\n }\n}\n", + "getParameter", + "protected_read", + ), + ( + "app.php", + " None: ("typescript", "function f(req: any) { const x = req.query.x; eval(x); }"), ("c", "void f(char *x) { char *y = x; system(y); }"), ("java", "class C { void f(String x) { String y = x; exec(y); } }"), + ("php", " None: @@ -99,8 +100,8 @@ def test_learn_slice_same_line_assignment_order() -> None: def test_learn_slice_fixed_framing_redaction_and_unsupported() -> None: - assert function_slice(" No def test_learn_slice_prompt_not_analysable() -> None: import test_learn_program as baseline - candidate = replace(baseline.candidate(), language="php") + candidate = replace(baseline.candidate(), language="ruby") prompt, _ = baseline.program(slice=True).prepare(candidate, baseline.FOLD) tail = str(prompt.messages[1]["content"]).split("Candidate:\n")[1] - assert candidate.code.rstrip() + "\nData flow inside the function: not analysed (php is not supported).\nSignals:" in tail + assert candidate.code.rstrip() + "\nData flow inside the function: not analysed (ruby is not supported).\nSignals:" in tail diff --git a/tests/test_obligation_facts.py b/tests/test_obligation_facts.py index c13b054..877218d 100644 --- a/tests/test_obligation_facts.py +++ b/tests/test_obligation_facts.py @@ -70,3 +70,24 @@ def test_flow_facts_loader_ignores_the_obligation_facts_directory() -> None: assert DEFAULT_OBLIGATION_FACTS_DIR.is_dir() and DEFAULT_OBLIGATION_FACTS_DIR.name == "obligations" flow = load_facts() assert not any(source.language == "obligations" for source in flow.sources) + + +@pytest.mark.parametrize( + ("language", "text", "operation", "discharger"), + [ + ("java", "class App { void run() { checkPermission(ADMINISTER); save(); } }", "jenkins-item-write", "jenkins-permission-guard"), + ("php", " None: + from openultrasast.semantic.facts import load_facts + from openultrasast.semantic.ir import parse_file + from openultrasast.semantic.obligations.facts import load_obligation_facts + from openultrasast.semantic.obligations.operations import find_discharges, find_operations + + facts = load_obligation_facts().for_language(language) + assert facts.operations and facts.dischargers + ir = parse_file(f"app.{language}", text, language) + assert ir.parse_ok + assert operation in {op.fact_id for op in find_operations(ir, facts, text=text)} + assert discharger in {d.fact_id for d in find_discharges(ir, facts, load_facts().for_language(language), text=text)} diff --git a/tests/test_semantic_cst.py b/tests/test_semantic_cst.py index 7c9e4f1..843b304 100644 --- a/tests/test_semantic_cst.py +++ b/tests/test_semantic_cst.py @@ -68,3 +68,72 @@ def test_c_constant_system_demotes(tmp_path: Path) -> None: assert demoted assert demoted[0].dominating_fact == "constant" assert demoted[0].evidence_level == "static_corroboration" + + +def test_php_functions_binds_calls_and_parameters(tmp_path: Path) -> None: + _require_extra() + from tree_sitter import Parser + + from openultrasast.semantic.cst import _KIND, _walk + from openultrasast.semantic.extra import grammar_for + + text = """

Text mode

+g($y); + $value .= "hello $x"; + $allowed = Gate::authorize('read', $x); + $this?->g($y); + } +} +function free_fn($x) { return consume($x); } +$closure = function ($z) { return consume($z); }; +$arrow = fn($a) => consume($a); +""" + path = tmp_path / "sample.php" + path.write_text(text) + data = path.read_bytes() + assert len(data) == len(text.encode()) > 0 + tree = Parser(grammar_for("php")).parse(data) + assert not tree.root_node.has_error + seen = {node.type for node in _walk(tree.root_node)} + # Exercise every registered kind against the installed grammar, including fn. + assert set().union(*_KIND["php"].values()) <= seen + ir = parse_file(path.name, data.decode(), "php") + assert ir.parse_ok and ir.engine == "tree-sitter" + assert [(fn.name, fn.start_line, fn.end_line, fn.params) for fn in ir.functions] == [ + ("", 1, 14, ()), + ("run", 4, 9, ("$x", "$y", "$rest")), + ("free_fn", 11, 11, ("$x",)), + ("", 12, 12, ("$z",)), + ("", 13, 13, ("$a",)), + ] + module, method, free, closure, arrow = ir.functions + assert module.calls == () + assert [b.name for b in module.binds] == ["$closure", "$arrow"] + assert [(b.name, b.line, b.names, b.call_name) for b in method.binds] == [ + ("$value", 5, ("$this", "$y"), "g"), + ("$value", 6, ("$value", "$x"), None), + ("$allowed", 7, ("$x",), "authorize"), + ] + assert [(c.name, c.line, c.arg_texts, c.arg_names) for c in method.calls] == [ + ("g", 5, ("$y",), (("$y",),)), + ("authorize", 7, ("'read'", "$x"), ((), ("$x",))), + ("g", 8, ("$y",), (("$y",),)), + ] + assert method.calls[1].arg_is_constant == (True, False) + for fn in (free, closure, arrow): + assert [call.name for call in fn.calls] == ["consume"] + assert fn.calls[0].arg_names == (fn.params,) + + +def test_php_interpolation_and_named_arguments() -> None: + _require_extra() + ir = parse_file("sample.php", ' None: assert "tree-sitter-javascript" in joined assert "tree-sitter-c" in joined assert "tree-sitter-java" in joined + assert "tree-sitter-php>=0.23" in extra assert "semantic:" in " ".join(payload["tool"]["pytest"]["ini_options"]["markers"])