mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-14 02:08:18 +00:00
feat(access-control-prover): migrate reviewed entry packs
This commit is contained in:
Marius Ciepluch
@@ -1,51 +1,267 @@
|
||||
"""Entry-fact packs loaded through the core fact-pack contract.
|
||||
|
||||
Packs live in ``ruleset/entries/*.json`` and are admitted only with a self-consistent
|
||||
review attestation (`core.factpack.load_pack_file`). Languages map pack languages to
|
||||
source languages; route patterns become extra recognition regexes.
|
||||
"""
|
||||
"""Installed entry-fact authority and projection-only legacy recognition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import cast
|
||||
|
||||
from openultrasast.core.factpack import load_pack_file
|
||||
from openultrasast.core import canonical, provenance
|
||||
from openultrasast.mapping.authority_ceremony import (
|
||||
INSTALLED_ARTIFACT_STORE_IDENTITY,
|
||||
load_installed_authority_ceremony,
|
||||
validate_authority_ceremony_bindings,
|
||||
)
|
||||
from openultrasast.mapping.fact_authority import (
|
||||
EntryFactAuthorityContext,
|
||||
authenticate_entry_fact_authority,
|
||||
)
|
||||
from openultrasast.mapping.installed_fact_authority import (
|
||||
PACK_FILES,
|
||||
DigestDirectoryResolver,
|
||||
InstalledAuthorityWireError,
|
||||
decode_fact_pack_bytes,
|
||||
decode_installed_authority_bytes,
|
||||
)
|
||||
from openultrasast.semantic.obligations.core_contract import production_access_control_contract
|
||||
|
||||
RECOGNITION_PROJECTION_SCHEMA = "openultrasast.entry-recognition-projection.v1"
|
||||
_PROJECTION_FIELDS = {"schema_version", "facts"}
|
||||
_ROW_FIELDS = {"legacy_fact_id", "languages", "defines_handler", "handler_kind", "pattern"}
|
||||
_LANGUAGE_ALIASES = {"js": "javascript", "ts": "typescript"}
|
||||
_cache: dict[tuple[str, tuple[tuple[str, float], ...]], dict[str, list[re.Pattern[str]]]] = {}
|
||||
_STATE_FILES = (
|
||||
"bootstrap.json",
|
||||
"identities/generator.json",
|
||||
"identities/maintainer-reviewer.json",
|
||||
"identities/custodian.json",
|
||||
"authority.json",
|
||||
*PACK_FILES,
|
||||
"express.json",
|
||||
)
|
||||
_CACHE_LIMIT = 8
|
||||
_FileIdentity = tuple[str, int, int, int, int, int, str]
|
||||
_InstalledState = tuple[_FileIdentity, ...]
|
||||
_CachedPatterns = tuple[tuple[str, tuple[re.Pattern[str], ...]], ...]
|
||||
_cache: OrderedDict[str, tuple[_InstalledState, _CachedPatterns]] = OrderedDict()
|
||||
_cache_lock = Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RecognitionRow:
|
||||
legacy_fact_id: str
|
||||
languages: tuple[str, ...]
|
||||
defines_handler: bool
|
||||
handler_kind: str
|
||||
pattern: str
|
||||
|
||||
|
||||
class _ProjectionOnlyCustody:
|
||||
"""Authority authentication port that refuses evaluation-state use."""
|
||||
|
||||
def current_log(self, population_digest: str) -> provenance.PopulationCustodyLog:
|
||||
raise AssertionError(f"projection loading consumed target custody: {population_digest}")
|
||||
|
||||
|
||||
def _package_root(repo_root: Path | None) -> Path:
|
||||
if repo_root is not None:
|
||||
return repo_root
|
||||
import openultrasast
|
||||
|
||||
return Path(openultrasast.__file__).resolve().parent
|
||||
|
||||
|
||||
def _entries_root(repo_root: Path | None) -> Path:
|
||||
return _package_root(repo_root) / "ruleset" / "entries"
|
||||
|
||||
|
||||
def _state_paths(entries: Path) -> tuple[Path, ...]:
|
||||
fixed = tuple(entries / relative for relative in _STATE_FILES)
|
||||
artifact_root = entries / "artifacts" / "sha256"
|
||||
try:
|
||||
artifacts = tuple(sorted(path for path in artifact_root.rglob("*") if path.is_file()))
|
||||
except OSError as exc:
|
||||
raise InstalledAuthorityWireError("installed artifact state is unavailable") from exc
|
||||
return (*fixed, *artifacts)
|
||||
|
||||
|
||||
def _file_identity(entries: Path, path: Path) -> _FileIdentity:
|
||||
try:
|
||||
before = path.lstat()
|
||||
content = path.read_bytes()
|
||||
after = path.lstat()
|
||||
except OSError as exc:
|
||||
raise InstalledAuthorityWireError("installed authority state is unavailable") from exc
|
||||
before_metadata = (before.st_dev, before.st_ino, before.st_mode, before.st_size, before.st_mtime_ns)
|
||||
after_metadata = (after.st_dev, after.st_ino, after.st_mode, after.st_size, after.st_mtime_ns)
|
||||
if before_metadata != after_metadata:
|
||||
raise InstalledAuthorityWireError("installed authority state changed while fingerprinting")
|
||||
return (
|
||||
path.relative_to(entries).as_posix(),
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_mode,
|
||||
before.st_size,
|
||||
before.st_mtime_ns,
|
||||
canonical.bytes_digest(content),
|
||||
)
|
||||
|
||||
|
||||
def _installed_state(entries: Path) -> _InstalledState:
|
||||
paths = _state_paths(entries)
|
||||
state = tuple(_file_identity(entries, path) for path in paths)
|
||||
if paths != _state_paths(entries):
|
||||
raise InstalledAuthorityWireError("installed artifact set changed while fingerprinting")
|
||||
return state
|
||||
|
||||
|
||||
def _cached_patterns(entries: Path, state: _InstalledState) -> dict[str, list[re.Pattern[str]]] | None:
|
||||
key = str(entries.resolve())
|
||||
with _cache_lock:
|
||||
cached = _cache.get(key)
|
||||
if cached is None or cached[0] != state:
|
||||
return None
|
||||
_cache.move_to_end(key)
|
||||
return {language: list(patterns) for language, patterns in cached[1]}
|
||||
|
||||
|
||||
def _remember_patterns(entries: Path, state: _InstalledState, patterns: dict[str, list[re.Pattern[str]]]) -> None:
|
||||
key = str(entries.resolve())
|
||||
frozen = tuple((language, tuple(values)) for language, values in sorted(patterns.items()))
|
||||
with _cache_lock:
|
||||
_cache[key] = (state, frozen)
|
||||
_cache.move_to_end(key)
|
||||
while len(_cache) > _CACHE_LIMIT:
|
||||
_cache.popitem(last=False)
|
||||
|
||||
|
||||
def load_installed_entry_fact_authority(
|
||||
custody: provenance.PopulationCustodyPort,
|
||||
repo_root: Path | None = None,
|
||||
) -> EntryFactAuthorityContext:
|
||||
"""Decode and externally authenticate the closed installed v2 authority.
|
||||
|
||||
``custody`` remains caller supplied. Authority authentication verifies its port
|
||||
shape but deliberately does not read evaluation population state.
|
||||
"""
|
||||
entries = _entries_root(repo_root)
|
||||
contract = production_access_control_contract()
|
||||
try:
|
||||
packs = tuple(decode_fact_pack_bytes((entries / name).read_bytes()) for name in PACK_FILES)
|
||||
bundle = decode_installed_authority_bytes((entries / "authority.json").read_bytes(), packs, contract)
|
||||
except OSError as exc:
|
||||
raise InstalledAuthorityWireError("installed entry authority is unavailable") from exc
|
||||
ceremony = load_installed_authority_ceremony(entries)
|
||||
validate_authority_ceremony_bindings(ceremony, bundle)
|
||||
resolver = DigestDirectoryResolver(entries, INSTALLED_ARTIFACT_STORE_IDENTITY)
|
||||
return authenticate_entry_fact_authority(bundle, ceremony.bootstrap, custody, resolver)
|
||||
|
||||
|
||||
def _text(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise InstalledAuthorityWireError(f"{field} must be nonempty text")
|
||||
return value
|
||||
|
||||
|
||||
def _decode_projection(content: bytes) -> tuple[_RecognitionRow, ...]:
|
||||
try:
|
||||
raw = json.loads(content)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise InstalledAuthorityWireError("recognition projection is not canonical JSON") from exc
|
||||
if not isinstance(raw, dict) or set(raw) != _PROJECTION_FIELDS:
|
||||
raise InstalledAuthorityWireError("recognition projection has unknown or missing fields")
|
||||
if raw.get("schema_version") != RECOGNITION_PROJECTION_SCHEMA:
|
||||
raise InstalledAuthorityWireError("recognition projection schema is unknown")
|
||||
facts = raw.get("facts")
|
||||
if not isinstance(facts, list) or not facts:
|
||||
raise InstalledAuthorityWireError("recognition projection facts must be a nonempty list")
|
||||
rows: list[_RecognitionRow] = []
|
||||
for item in facts:
|
||||
if not isinstance(item, dict) or set(item) != _ROW_FIELDS:
|
||||
raise InstalledAuthorityWireError("recognition projection row has unknown or missing fields")
|
||||
languages = item["languages"]
|
||||
if (
|
||||
not isinstance(languages, list)
|
||||
or not languages
|
||||
or not all(isinstance(language, str) and language for language in languages)
|
||||
or len(set(languages)) != len(languages)
|
||||
):
|
||||
raise InstalledAuthorityWireError("recognition projection languages are invalid")
|
||||
defines_handler = item["defines_handler"]
|
||||
if not isinstance(defines_handler, bool):
|
||||
raise InstalledAuthorityWireError("recognition projection defines_handler must be boolean")
|
||||
rows.append(
|
||||
_RecognitionRow(
|
||||
_text(item["legacy_fact_id"], "legacy_fact_id"),
|
||||
tuple(languages),
|
||||
defines_handler,
|
||||
_text(item["handler_kind"], "handler_kind"),
|
||||
_text(item["pattern"], "pattern"),
|
||||
)
|
||||
)
|
||||
record = cast(
|
||||
Mapping[str, canonical.JsonValue],
|
||||
{
|
||||
"schema_version": RECOGNITION_PROJECTION_SCHEMA,
|
||||
"facts": tuple(
|
||||
{
|
||||
"legacy_fact_id": row.legacy_fact_id,
|
||||
"languages": row.languages,
|
||||
"defines_handler": row.defines_handler,
|
||||
"handler_kind": row.handler_kind,
|
||||
"pattern": row.pattern,
|
||||
}
|
||||
for row in rows
|
||||
),
|
||||
},
|
||||
)
|
||||
expected = canonical.canonical_record_bytes(record, known_schema_versions=(RECOGNITION_PROJECTION_SCHEMA,))
|
||||
if content != expected:
|
||||
raise InstalledAuthorityWireError("recognition projection does not round-trip canonically")
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def load_entry_pack_patterns(repo_root: Path | None = None) -> dict[str, list[re.Pattern[str]]]:
|
||||
"""Compiled route patterns per source language, from admitted entry packs.
|
||||
"""Return legacy recognition regexes from an authenticated projection only.
|
||||
|
||||
Packs resolve from the installed package's ``ruleset/entries`` (not the analyzed
|
||||
repository), so recognition is active on external repos; ``repo_root`` is accepted
|
||||
only for tests that stage a fake ruleset.
|
||||
This compatibility API intentionally returns no ``FactPack``, structural
|
||||
requirement, relation effect, or readiness value. A projection hit therefore
|
||||
cannot be promoted into detection-core evidence.
|
||||
"""
|
||||
import openultrasast
|
||||
|
||||
base = repo_root if repo_root is not None else Path(openultrasast.__file__).resolve().parent
|
||||
entries = base / "ruleset" / "entries"
|
||||
entries = _entries_root(repo_root)
|
||||
if not entries.is_dir():
|
||||
return {}
|
||||
files = sorted(entries.glob("*.json"))
|
||||
stamp = tuple((p.name, p.stat().st_mtime) for p in files)
|
||||
key = (str(entries.resolve()), stamp)
|
||||
if key in _cache:
|
||||
return _cache[key]
|
||||
state = _installed_state(entries)
|
||||
cached = _cached_patterns(entries, state)
|
||||
if cached is not None:
|
||||
return cached
|
||||
context = load_installed_entry_fact_authority(_ProjectionOnlyCustody(), repo_root)
|
||||
resolved = context.resolver.resolve(context.bundle.recognition_projection).content
|
||||
try:
|
||||
installed = (entries / "express.json").read_bytes()
|
||||
except OSError as exc:
|
||||
raise InstalledAuthorityWireError("installed recognition projection is unavailable") from exc
|
||||
if installed != resolved:
|
||||
raise InstalledAuthorityWireError("compatibility projection differs from authenticated artifact")
|
||||
patterns: dict[str, list[re.Pattern[str]]] = {}
|
||||
for pack_file in files:
|
||||
payload = json.loads(pack_file.read_text(encoding="utf-8"))
|
||||
pack = load_pack_file(pack_file)
|
||||
for raw_language in payload.get("languages", []):
|
||||
for row in _decode_projection(installed):
|
||||
if row.handler_kind != "route" or not row.defines_handler:
|
||||
continue
|
||||
for raw_language in row.languages:
|
||||
language = _LANGUAGE_ALIASES.get(str(raw_language), str(raw_language))
|
||||
for fact in pack.facts:
|
||||
if fact.get("handler_kind") == "route" and fact.get("pattern"):
|
||||
patterns.setdefault(language, []).append(re.compile(str(fact["pattern"])))
|
||||
_cache[key] = patterns
|
||||
patterns.setdefault(language, []).append(re.compile(row.pattern))
|
||||
if state != _installed_state(entries):
|
||||
raise InstalledAuthorityWireError("installed authority state changed during authentication")
|
||||
_remember_patterns(entries, state, patterns)
|
||||
return patterns
|
||||
|
||||
|
||||
__all__ = ["load_entry_pack_patterns"]
|
||||
__all__ = [
|
||||
"RECOGNITION_PROJECTION_SCHEMA",
|
||||
"load_entry_pack_patterns",
|
||||
"load_installed_entry_fact_authority",
|
||||
]
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"facts":[{"defines_handler":true,"handler_kind":"route","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:app-route","pattern":"app.(get|post|put|delete|patch|all|use)\\s*\\("},{"defines_handler":true,"handler_kind":"route","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:router-route","pattern":"router.(get|post|put|delete|patch|all|use)\\s*\\("},{"defines_handler":false,"handler_kind":"app","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:express-app","pattern":"express\\s*\\(\\s*\\)"}],"schema_version":"openultrasast.entry-recognition-projection.v1"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"conflicts":[],"resolved":true,"schema_version":"openultrasast.access-control.authority-conflict-disclosure.v1"}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"defines_handler": true,
|
||||
"handler_kind": "route",
|
||||
"id": "entry:express:app-route",
|
||||
"pattern": "app.(get|post|put|delete|patch|all|use)\\s*\\("
|
||||
},
|
||||
{
|
||||
"defines_handler": true,
|
||||
"handler_kind": "route",
|
||||
"id": "entry:express:router-route",
|
||||
"pattern": "router.(get|post|put|delete|patch|all|use)\\s*\\("
|
||||
},
|
||||
{
|
||||
"defines_handler": false,
|
||||
"handler_kind": "app",
|
||||
"id": "entry:express:express-app",
|
||||
"pattern": "express\\s*\\(\\s*\\)"
|
||||
}
|
||||
],
|
||||
"family": "access_control",
|
||||
"framework": "express",
|
||||
"kind": "entry_facts",
|
||||
"languages": [
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"review": {
|
||||
"digest": "47866e5c15b286d7107b101bd396f78bbbda23788085676c590c3af937085e00",
|
||||
"note": "Reviewed against Express 4/5 routing docs and the pilot pool's manifest-level framework usage (2026-10-30 recognition table). Entry patterns only; no sinks, no labels."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"authority_review":{"attestation_digest":"aeeb9120c10d8bf83159a9cbd14ad8a070a0ceb99a53b5a1daa616a315c84b55","conflict_disclosure":{"artifact_id":"installed-entry-authority-conflicts-v1","byte_length":113,"content_digest":"a8bf581d321560f2730aae3d5a7c9114c2d8f81f7a702fa3681a87e7be4cb9e1","media_type":"application/json","retention_policy_digest":"09a0fe8a253780b3666ef016d575a212bf0570da392dd5bf1a834dce2edf703f","storage_receipt_digest":"ee92fde020e9b4d316ad285f3f904032c52d902fe1c7c0a94cb0d61d9c0136ce"},"producer_identity":"9635025424be37e1615e597185a425a5443242fa210d6136c38250101088e7b4","reviewer_identity":"f12a7e897cbaf3059619517e1f8a725b90049a0edf3af08efa12fb1ef7fdee43","subject_digest":"0fa8f607297aa5364a0ca6c9b8d47bfcdef28456b7f062ddb251f8f8b9a67da4"},"bundle_digest":"0fa8f607297aa5364a0ca6c9b8d47bfcdef28456b7f062ddb251f8f8b9a67da4","custody_policy":{"policy_digest":"415b1c13adc918c162932a38e79548a7d9867f5331860eb409238c51b1cca1f0","reviewed_custodian_digests":["4fe8ca2923ee759a24fd8e5b91673faa6859a7a0c2df6e7c7ea5e23968708632"]},"lineages":[{"arm_id":"installed-entry-facts:javascript","manifest_digest":"f3b4b192dadd6b7d0d298286008480dadfd8640e9603020b0f24d13dce9bda04","subject_digest":"331e8c2261cb030f4257a8650448560eb94941554df47cb275882e24b8f8923f","uses":[{"influenced_artifact_digest":"331e8c2261cb030f4257a8650448560eb94941554df47cb275882e24b8f8923f","labels_accessed":false,"population_use_receipt_digest":"36ccebd06b60601775ab24a2f7ec9f2b49c2eac6b273816eaf9e867f6c445915","purpose":"semantic_fact","source_population_digest":"0ec4bbe3712a691a76f6525aca59f15d0492adc1e7ec8d7e9bb42e8ee42795ce"}]},{"arm_id":"installed-entry-facts:typescript","manifest_digest":"b7a6a95ddcdd8f6e6dbcd2b90cb707eda9833ec7479cb4d7aae3c91cfe0059ca","subject_digest":"b6d1157e99537bd4928303ec0d887628bc53f0aa9e770427580a32e3948babd9","uses":[{"influenced_artifact_digest":"b6d1157e99537bd4928303ec0d887628bc53f0aa9e770427580a32e3948babd9","labels_accessed":false,"population_use_receipt_digest":"7dc5fe36b1e111ab6cf979137c40bf67b885b2a9deda944aff78e3031cb5a553","purpose":"semantic_fact","source_population_digest":"0ec4bbe3712a691a76f6525aca59f15d0492adc1e7ec8d7e9bb42e8ee42795ce"}]},{"arm_id":"installed-entry-structural-matcher","manifest_digest":"5f7ffd58a773b61d1f8e2ba0ca019e27d778166e13cec770e6f25c16b1f1a994","subject_digest":"0ad18521beeb3708e12f8c9941b3f29b3e1312fe78b9320e5aaf15bc9db8831a","uses":[{"influenced_artifact_digest":"0ad18521beeb3708e12f8c9941b3f29b3e1312fe78b9320e5aaf15bc9db8831a","labels_accessed":false,"population_use_receipt_digest":"a602c8bccc6afc2ff9d857119131a2847509bc693a80192bf8459b9dcba9cbda","purpose":"matching_behavior","source_population_digest":"0ec4bbe3712a691a76f6525aca59f15d0492adc1e7ec8d7e9bb42e8ee42795ce"}]}],"matcher_behavior_digest":"0ad18521beeb3708e12f8c9941b3f29b3e1312fe78b9320e5aaf15bc9db8831a","matcher_lineage_manifest_digest":"5f7ffd58a773b61d1f8e2ba0ca019e27d778166e13cec770e6f25c16b1f1a994","matcher_revision":"entry-structural-matcher-v1","pack_files":["express.javascript.v2.json","express.typescript.v2.json"],"recognition_projection":{"artifact_id":"express-recognition-projection-v1","byte_length":628,"content_digest":"529d18ba2b351a8de985a9cf416261d717ef55a4b7a9d9d65bce53b97d59ab35","media_type":"application/json","retention_policy_digest":"09a0fe8a253780b3666ef016d575a212bf0570da392dd5bf1a834dce2edf703f","storage_receipt_digest":"3a85c0641cddcfc8c926fd01e230360c042606bda78e14a39489de14f88944b4"},"review_policy":{"forbidden_identity_pairs":[],"maximum_evidence_bytes":1048576,"policy_digest":"cb023a4f7e727ec4abfee583256194fe3f11c064a10960f297c1f7bf2e6fecde","reviewed_reviewer_digests":["f12a7e897cbaf3059619517e1f8a725b90049a0edf3af08efa12fb1ef7fdee43"],"trusted_store_identities":["openultrasast-installed-entry-artifacts-v1"]},"schema_version":"openultrasast.access-control.installed-entry-authority.v1"}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,33 +1 @@
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"defines_handler": true,
|
||||
"handler_kind": "route",
|
||||
"id": "entry:express:app-route",
|
||||
"pattern": "app.(get|post|put|delete|patch|all|use)\\s*\\("
|
||||
},
|
||||
{
|
||||
"defines_handler": true,
|
||||
"handler_kind": "route",
|
||||
"id": "entry:express:router-route",
|
||||
"pattern": "router.(get|post|put|delete|patch|all|use)\\s*\\("
|
||||
},
|
||||
{
|
||||
"defines_handler": false,
|
||||
"handler_kind": "app",
|
||||
"id": "entry:express:express-app",
|
||||
"pattern": "express\\s*\\(\\s*\\)"
|
||||
}
|
||||
],
|
||||
"family": "access_control",
|
||||
"framework": "express",
|
||||
"kind": "entry_facts",
|
||||
"languages": [
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"review": {
|
||||
"digest": "47866e5c15b286d7107b101bd396f78bbbda23788085676c590c3af937085e00",
|
||||
"note": "Reviewed against Express 4/5 routing docs and the pilot pool's manifest-level framework usage (2026-10-30 recognition table). Entry patterns only; no sinks, no labels."
|
||||
}
|
||||
}
|
||||
{"facts":[{"defines_handler":true,"handler_kind":"route","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:app-route","pattern":"app.(get|post|put|delete|patch|all|use)\\s*\\("},{"defines_handler":true,"handler_kind":"route","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:router-route","pattern":"router.(get|post|put|delete|patch|all|use)\\s*\\("},{"defines_handler":false,"handler_kind":"app","languages":["javascript","typescript"],"legacy_fact_id":"entry:express:express-app","pattern":"express\\s*\\(\\s*\\)"}],"schema_version":"openultrasast.entry-recognition-projection.v1"}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user