feat(access-control-prover): publish fact conformance handoff

This commit is contained in:
Marius Ciepluch
2026-11-01 18:16:07 +01:00
parent ec6c8f7f3e
commit 29030b38fc
37 changed files with 2333 additions and 1 deletions
@@ -0,0 +1,128 @@
"""No-follow filesystem primitives for the conformance handoff."""
from __future__ import annotations
import os
import stat
from pathlib import Path
class ConformanceFilesystemError(ValueError):
"""A handoff path was redirected or was not the required node type."""
def _open_flags(*flags: int) -> int:
result = 0
for flag in flags:
result |= flag
return result
def _read_fd(fd: int) -> bytes:
chunks: list[bytes] = []
while chunk := os.read(fd, 1024 * 1024):
chunks.append(chunk)
return b"".join(chunks)
def read_regular_nofollow(path: Path) -> bytes:
"""Read one regular file without following its final path component."""
flags = _open_flags(os.O_RDONLY, os.O_NOFOLLOW, os.O_NONBLOCK)
try:
fd = os.open(path, flags)
except OSError as exc:
raise ConformanceFilesystemError("path is not a readable regular file") from exc
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise ConformanceFilesystemError("path is not a regular file")
return _read_fd(fd)
except OSError as exc:
raise ConformanceFilesystemError("regular file could not be read") from exc
finally:
os.close(fd)
def require_real_directory(path: Path) -> None:
"""Require an existing directory whose final component is not a symlink."""
flags = _open_flags(os.O_RDONLY, os.O_DIRECTORY, os.O_NOFOLLOW)
try:
fd = os.open(path, flags)
except OSError as exc:
raise ConformanceFilesystemError("path is not a real directory") from exc
try:
if not stat.S_ISDIR(os.fstat(fd).st_mode):
raise ConformanceFilesystemError("path is not a real directory")
finally:
os.close(fd)
def ensure_real_directory(path: Path) -> None:
"""Create a directory if absent, then reject redirected final components."""
try:
path.mkdir(parents=True, exist_ok=True)
except OSError as exc:
raise ConformanceFilesystemError("directory could not be created") from exc
require_real_directory(path)
def flat_regular_census(path: Path) -> tuple[str, ...]:
"""Return direct regular-file names and reject every other entry.
Whole-directory validation assumes no concurrent writer. Opening both the
directory and subsequently read files with ``O_NOFOLLOW`` closes the usual
check-to-open symlink race for each operation.
"""
flags = _open_flags(os.O_RDONLY, os.O_DIRECTORY, os.O_NOFOLLOW)
try:
fd = os.open(path, flags)
except OSError as exc:
raise ConformanceFilesystemError("census root is not a real directory") from exc
try:
if not stat.S_ISDIR(os.fstat(fd).st_mode):
raise ConformanceFilesystemError("census root is not a real directory")
with os.scandir(fd) as entries:
names = []
for entry in entries:
if not entry.is_file(follow_symlinks=False) or entry.is_symlink():
raise ConformanceFilesystemError("census contains an unsafe entry")
names.append(entry.name)
return tuple(sorted(names))
except OSError as exc:
raise ConformanceFilesystemError("directory census failed") from exc
finally:
os.close(fd)
def persist_regular_nofollow(path: Path, content: bytes) -> None:
"""Create an immutable-addressed file or verify an existing regular file."""
flags = _open_flags(os.O_CREAT, os.O_EXCL, os.O_WRONLY, os.O_NOFOLLOW)
try:
fd = os.open(path, flags, 0o644)
except FileExistsError:
if read_regular_nofollow(path) != content:
raise ConformanceFilesystemError("existing digest file has different bytes") from None
return
except OSError as exc:
raise ConformanceFilesystemError("digest file could not be created") from exc
try:
view = memoryview(content)
while view:
written = os.write(fd, view)
if written <= 0:
raise ConformanceFilesystemError("digest file write made no progress")
view = view[written:]
os.fsync(fd)
except OSError as exc:
raise ConformanceFilesystemError("digest file could not be written") from exc
finally:
os.close(fd)
__all__ = [
"ConformanceFilesystemError",
"ensure_real_directory",
"flat_regular_census",
"persist_regular_nofollow",
"read_regular_nofollow",
"require_real_directory",
]
@@ -0,0 +1,300 @@
"""Negative and replay cases for the downstream entry-fact handoff."""
from __future__ import annotations
from dataclasses import replace
from pathlib import Path
from typing import Any, NoReturn
from openultrasast.core import canonical, provenance
from openultrasast.core.factmatch import validate_fact_match
from openultrasast.core.factpack import derive_independence_attestation_digest
from openultrasast.mapping.authority_ceremony import load_installed_authority_ceremony
from openultrasast.mapping.entry_fact_matcher import EntryFactMatchResult, match_entry_fact
from openultrasast.mapping.entry_packs import load_entry_pack_patterns
from openultrasast.mapping.fact_authority import EntryFactAuthorityContext, authenticate_entry_fact_authority
from openultrasast.mapping.fact_match_store import FactMatchArtifactWriter
from openultrasast.mapping.installed_fact_authority import fact_pack_bytes, installed_authority_bytes
_AUTHENTICATE = "openultrasast.mapping.fact_authority.authenticate_entry_fact_authority"
def negative_cases(
fixture_root: Path,
output: Path,
sources: dict[str, tuple[str, bytes]],
context: EntryFactAuthorityContext,
evaluation: str,
complete: EntryFactMatchResult,
) -> tuple[dict[str, canonical.JsonValue], ...]:
from openultrasast.mapping.entry_fact_conformance import (
_CORE_ENTRY_POINTS,
_case,
_persist,
_validator,
)
common = (_persist(output, fact_pack_bytes(complete.pack), "fact-pack"),)
rows = _structural_refusals(fixture_root, output, sources, context, evaluation, common)
rows.extend(_authority_cases(output, sources, context))
rows.append(_resolver_case(sources["resolver_byte_substitution"], common, context, evaluation, complete))
rows.extend(_lineage_cases(sources, context, evaluation, complete, common))
rows.append(_contaminated_case(sources["custody_contamination"], common, context, evaluation, complete))
_legacy_only(fixture_root, output, sources, context, evaluation)
rows.append(
_case(
"legacy_projection_only",
sources["legacy_projection_only"],
"refused",
"name_only_match",
common,
(_validator(_CORE_ENTRY_POINTS[1], "refused", "name_only_match"),),
)
)
return tuple(rows)
def _authority_cases(
output: Path,
sources: dict[str, tuple[str, bytes]],
context: EntryFactAuthorityContext,
) -> tuple[dict[str, canonical.JsonValue], ...]:
from openultrasast.mapping.entry_fact_conformance import _case, _installed_entries, _persist, _validator
bundle = context.bundle
ceremony = load_installed_authority_ceremony(_installed_entries())
substituted = replace(bundle, bundle_digest="0" * 64)
_expect_error(
lambda: authenticate_entry_fact_authority(substituted, ceremony.bootstrap, context.custody, context.resolver),
"authority bundle digest",
)
substituted_case = _case(
"authority_substitution",
sources["authority_substitution"],
"refused",
"match_pack_identity_substituted",
(_persist(output, installed_authority_bytes(substituted), "authority"),),
(_validator("openultrasast.core.factmatch.validate_fact_match", "refused", "match_pack_identity_substituted"),),
(_validator(_AUTHENTICATE, "refused", "authority_bundle_digest_substituted"),),
)
review = replace(bundle.authority_review, producer_identity=bundle.authority_review.reviewer_identity, attestation_digest="0" * 64)
review = replace(review, attestation_digest=derive_independence_attestation_digest(review))
conflicted = replace(bundle, authority_review=review)
_expect_error(
lambda: authenticate_entry_fact_authority(conflicted, ceremony.bootstrap, context.custody, context.resolver),
"producer and reviewer",
)
conflict_case = _case(
"producer_reviewer_conflict",
sources["producer_reviewer_conflict"],
"refused",
"self_authored_review_evidence",
(_persist(output, installed_authority_bytes(conflicted), "authority"),),
(_validator("openultrasast.core.factpack.validate_fact_pack", "refused", "self_authored_review_evidence"),),
(_validator(_AUTHENTICATE, "refused", "producer_reviewer_conflict"),),
)
return substituted_case, conflict_case
def _resolver_case(
source: tuple[str, bytes],
artifacts: tuple[dict[str, canonical.JsonValue], ...],
context: EntryFactAuthorityContext,
evaluation: str,
complete: EntryFactMatchResult,
) -> dict[str, canonical.JsonValue]:
from openultrasast.mapping.entry_fact_conformance import _CORE_ENTRY_POINTS, _case, _SubstitutingResolver, _validator
bundle = context.bundle
target = complete.match.observations[0].evidence.content_digest
resolver = _SubstitutingResolver(complete.resolver, target)
_expect_error(
lambda: validate_fact_match(
complete.pack,
complete.match.fact_id,
complete.match,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
resolver,
),
"resolved content digest",
)
return _case(
"resolver_byte_substitution",
source,
"refused",
"resolver_byte_substitution",
artifacts,
(_validator(_CORE_ENTRY_POINTS[1], "refused", "resolver_byte_substitution"),),
)
def _structural_refusals(
root: Path,
output: Path,
sources: dict[str, tuple[str, bytes]],
context: EntryFactAuthorityContext,
evaluation: str,
artifacts: tuple[dict[str, canonical.JsonValue], ...],
) -> list[dict[str, canonical.JsonValue]]:
from openultrasast.mapping.entry_fact_conformance import _CORE_ENTRY_POINTS, _case, _match, _match_artifacts, _validator
rows = []
for case_id, reason in (("name_only", "missing_observation:receiver"), ("partial", "missing_observation:arguments")):
result = _match(root, output, context, evaluation, sources[case_id][0])
if result.decision.effect_eligible or reason not in result.decision.reasons:
_fail(f"{case_id} did not refuse with its stable reason")
case_artifacts = (*artifacts, *_match_artifacts(output, result))
rows.append(
_case(
case_id,
sources[case_id],
"refused",
reason,
case_artifacts,
(_validator(_CORE_ENTRY_POINTS[1], "refused", reason),),
)
)
return rows
def _lineage_cases(
sources: dict[str, tuple[str, bytes]],
context: EntryFactAuthorityContext,
evaluation: str,
complete: EntryFactMatchResult,
artifacts: tuple[dict[str, canonical.JsonValue], ...],
) -> tuple[dict[str, canonical.JsonValue], ...]:
from openultrasast.mapping.entry_fact_conformance import _CORE_ENTRY_POINTS, _case, _validator
bundle = context.bundle
omitted = tuple(item for item in bundle.lineages if item.manifest_digest != complete.match.matcher_lineage_manifest_digest)
_expect_error(
lambda: validate_fact_match(
complete.pack,
complete.match.fact_id,
complete.match,
evaluation,
omitted,
context.custody,
bundle.custody_policy,
bundle.review_policy,
complete.resolver,
),
"matcher lineage",
)
omitted_case = _case(
"lineage_omission",
sources["lineage_omission"],
"refused",
"matcher_lineage_omitted",
artifacts,
(_validator(_CORE_ENTRY_POINTS[1], "refused", "matcher_lineage_omitted"),),
)
replay = validate_fact_match(
complete.pack,
complete.match.fact_id,
complete.match,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
complete.resolver,
)
if not replay.effect_eligible:
_fail("deterministic replay did not remain complete")
replay_case = _case(
"deterministic_replay", sources["deterministic_replay"], "passed", None, artifacts, (_validator(_CORE_ENTRY_POINTS[1], "passed"),)
)
return omitted_case, replay_case
def _contaminated_case(
source: tuple[str, bytes],
artifacts: tuple[dict[str, canonical.JsonValue], ...],
context: EntryFactAuthorityContext,
evaluation: str,
complete: EntryFactMatchResult,
) -> dict[str, canonical.JsonValue]:
from openultrasast.mapping.entry_fact_conformance import _CORE_ENTRY_POINTS, _case, _Custody, _validator
custody = context.custody
if not isinstance(custody, _Custody):
_fail("conformance custody has the wrong type")
prior = custody.current_log(evaluation)
lineage = next(item for item in context.bundle.lineages if item.manifest_digest == complete.match.matcher_lineage_manifest_digest)
custody.append(
provenance.PopulationUseRequest(
evaluation,
lineage.arm_id,
provenance.LineagePurpose.MATCHING_BEHAVIOR,
True,
complete.match.matcher_behavior_digest,
prior.log_digest,
)
)
decision = validate_fact_match(
complete.pack,
complete.match.fact_id,
complete.match,
evaluation,
context.bundle.lineages,
custody,
context.bundle.custody_policy,
context.bundle.review_policy,
complete.resolver,
)
reason = next((item for item in decision.reasons if item.startswith("protected_population_labels:")), None)
if decision.effect_eligible or reason is None:
_fail("protected population contamination was not refused")
return _case(
"custody_contamination",
source,
"refused",
"protected_population_contamination",
artifacts,
(_validator(_CORE_ENTRY_POINTS[1], "refused", "protected_population_contamination"),),
)
def _legacy_only(
root: Path, output: Path, sources: dict[str, tuple[str, bytes]], context: EntryFactAuthorityContext, evaluation: str
) -> None:
content = sources["legacy_projection_only"][1].decode("utf-8")
patterns = load_entry_pack_patterns()["javascript"]
writer = FactMatchArtifactWriter(
output, context.bundle.review_policy.trusted_store_identities[0], canonical.content_digest("retain-access-control-conformance-v1")
)
matches = match_entry_fact(
repository_root=root,
relative_path=sources["legacy_projection_only"][0],
language="javascript",
fact_id="entry:express:app-route:get",
authority=context,
writer=writer,
evaluation_population_digest=evaluation,
)
if not any(pattern.search(content) for pattern in patterns) or matches:
_fail("legacy-only fixture did not isolate projection recognition")
def _expect_error(action: Any, fragment: str) -> None:
try:
action()
except (TypeError, ValueError) as exc:
if fragment not in str(exc):
_fail(f"validator refusal changed: {exc}", exc)
return
_fail("negative conformance case unexpectedly passed")
def _fail(message: str, cause: Exception | None = None) -> NoReturn:
from openultrasast.mapping.entry_fact_conformance import EntryFactConformanceError
if cause is not None:
raise EntryFactConformanceError(message) from cause
raise EntryFactConformanceError(message)
@@ -0,0 +1,96 @@
"""Materialize the self-sufficient generic-core replay artifact."""
from __future__ import annotations
from collections.abc import Mapping
from pathlib import Path
from openultrasast.core import canonical, provenance
from openultrasast.mapping._installed_fact_authority_codec import wire_value
from openultrasast.mapping.authority_ceremony import INSTALLED_ARTIFACT_STORE_IDENTITY
from openultrasast.mapping.entry_fact_matcher import EntryFactMatchResult
from openultrasast.mapping.fact_authority import EntryFactAuthorityContext
def _resolver_material(
output: Path,
result: EntryFactMatchResult,
) -> tuple[tuple[dict[str, canonical.JsonValue], ...], tuple[dict[str, canonical.JsonValue], ...], str]:
from openultrasast.mapping.entry_fact_conformance import _persist
refs = [item.content for item in result.pack.review_evidence]
refs.extend(item.independence.conflict_disclosure for item in result.pack.review_evidence)
refs.extend(item.evidence for item in result.match.observations)
refs.append(result.match_reference)
artifacts: list[dict[str, canonical.JsonValue]] = []
entries: list[dict[str, canonical.JsonValue]] = []
for reference in refs:
artifact = _persist(output, result.resolver.resolve(reference).content, "resolver-content")
artifacts.append(artifact)
entries.append(
{
"reference": wire_value(reference),
"path": artifact["path"],
"store_identity": INSTALLED_ARTIFACT_STORE_IDENTITY,
}
)
target = result.match.observations[0].evidence
original = result.resolver.resolve(target).content
substituted = _persist(output, b"!" + original[1:], "substituted-content")
artifacts.append(substituted)
return tuple(entries), tuple(artifacts), str(substituted["path"])
def replay_material(
output: Path,
context: EntryFactAuthorityContext,
evaluation: str,
result: EntryFactMatchResult,
source: tuple[str, bytes],
) -> tuple[dict[str, canonical.JsonValue], tuple[dict[str, canonical.JsonValue], ...]]:
from openultrasast.mapping._entry_fact_conformance_wire import replay_envelope_bytes
from openultrasast.mapping.entry_fact_conformance import EntryFactConformanceError, _Custody, _persist, _record
custody = context.custody
if not isinstance(custody, _Custody):
raise EntryFactConformanceError("conformance custody has the wrong type")
contaminated = custody.current_log(evaluation)
custodian = context.bundle.custody_policy.reviewed_custodian_digests[0]
base_evaluation = provenance.empty_population_custody_log(evaluation, custodian)
base_logs = tuple(base_evaluation if digest == evaluation else log for digest, log in sorted(custody.logs.items()))
entries, artifacts, substituted_path = _resolver_material(output, result)
record = _record(result, evaluation, canonical.bytes_digest(source[1]))
content = replay_envelope_bytes(
context=context,
evaluation_population_digest=evaluation,
result=result,
record=record,
base_logs=base_logs,
contaminated_log=contaminated,
resolver_entries=entries,
substituted_content_path=substituted_path,
)
return _persist(output, content, "core-replay-envelope"), artifacts
def attach_replay(
cases: tuple[dict[str, canonical.JsonValue], ...],
envelope: dict[str, canonical.JsonValue],
dependencies: tuple[dict[str, canonical.JsonValue], ...],
) -> tuple[dict[str, canonical.JsonValue], ...]:
values = []
for case in cases:
artifacts = case["artifacts"]
if not isinstance(artifacts, tuple):
raise TypeError("case artifacts must be a tuple")
if not all(isinstance(item, Mapping) for item in artifacts):
raise TypeError("case artifact entries must be objects")
existing = {str(item["path"]): item for item in artifacts if isinstance(item, Mapping)}
existing[str(envelope["path"])] = envelope
if case["case_id"] == "complete":
existing.update({str(item["path"]): item for item in dependencies})
values.append({**case, "artifacts": tuple(existing[path] for path in sorted(existing))})
return tuple(values)
__all__ = ["attach_replay", "replay_material"]
@@ -0,0 +1,70 @@
"""Canonical generic-core replay envelope for the entry-fact handoff."""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from openultrasast.core import canonical, evidence, provenance
from openultrasast.mapping._installed_fact_authority_codec import wire_value
from openultrasast.mapping.entry_fact_matcher import EntryFactMatchResult
from openultrasast.mapping.fact_authority import EntryFactAuthorityContext
ENTRY_FACT_REPLAY_SCHEMA = "openultrasast.access-control.entry-fact-core-replay.v1"
def _case_plans() -> tuple[Mapping[str, canonical.JsonValue], ...]:
match = "openultrasast.core.factmatch.validate_fact_match"
return (
{"case_id": "authority_substitution", "replay_kind": "core", "validator": match, "mutation": "match_pack_digest_substitution"},
{"case_id": "complete", "replay_kind": "core", "validator": "all", "mutation": "none"},
{"case_id": "custody_contamination", "replay_kind": "core", "validator": match, "mutation": "contaminated_custody"},
{"case_id": "deterministic_replay", "replay_kind": "core", "validator": match, "mutation": "none"},
{"case_id": "legacy_projection_only", "replay_kind": "core", "validator": match, "mutation": "qualified_name_only"},
{"case_id": "lineage_omission", "replay_kind": "core", "validator": match, "mutation": "matcher_lineage_omission"},
{"case_id": "name_only", "replay_kind": "core", "validator": match, "mutation": "qualified_name_only"},
{"case_id": "partial", "replay_kind": "core", "validator": match, "mutation": "arguments_omission"},
{
"case_id": "producer_reviewer_conflict",
"replay_kind": "core",
"validator": "openultrasast.core.factpack.validate_fact_pack",
"mutation": "self_authored_pack",
},
{"case_id": "resolver_byte_substitution", "replay_kind": "core", "validator": match, "mutation": "resolver_byte_substitution"},
)
def replay_envelope_bytes(
*,
context: EntryFactAuthorityContext,
evaluation_population_digest: str,
result: EntryFactMatchResult,
record: evidence.EvidenceRecord,
base_logs: Sequence[provenance.PopulationCustodyLog],
contaminated_log: provenance.PopulationCustodyLog,
resolver_entries: Sequence[Mapping[str, canonical.JsonValue]],
substituted_content_path: str,
) -> bytes:
"""Serialize every family-neutral input needed for independent core replay."""
payload: Mapping[str, canonical.JsonValue] = {
"schema_version": ENTRY_FACT_REPLAY_SCHEMA,
"contract": wire_value(context.bundle.contract),
"pack": wire_value(result.pack),
"fact_id": result.match.fact_id,
"match": wire_value(result.match),
"match_reference": wire_value(result.match_reference),
"evidence_record": wire_value(record),
"evaluation_population_digest": evaluation_population_digest,
"lineages": wire_value(context.bundle.lineages),
"review_policy": wire_value(context.bundle.review_policy),
"custody_policy": wire_value(context.bundle.custody_policy),
"custody_logs": wire_value(tuple(base_logs)),
"contaminated_custody_log": wire_value(contaminated_log),
"resolver_entries": tuple(resolver_entries),
"substituted_content_path": substituted_content_path,
"case_plans": tuple(_case_plans()),
}
envelope = {**payload, "envelope_digest": canonical.content_digest(payload)}
return canonical.canonical_record_bytes(envelope, known_schema_versions=(ENTRY_FACT_REPLAY_SCHEMA,))
__all__ = ["ENTRY_FACT_REPLAY_SCHEMA", "replay_envelope_bytes"]
@@ -0,0 +1,390 @@
"""Reproducible downstream conformance handoff for reviewed entry facts."""
from __future__ import annotations
from dataclasses import dataclass, replace
from pathlib import Path
from openultrasast.core import canonical, evidence, family, provenance, semantics
from openultrasast.core.factapply import apply_fact
from openultrasast.core.factmatch import validate_fact_match
from openultrasast.core.factpack import validate_fact_pack
from openultrasast.core.factvalidate import validate_fact_relations
from openultrasast.mapping._entry_fact_conformance_fs import (
ConformanceFilesystemError,
ensure_real_directory,
persist_regular_nofollow,
read_regular_nofollow,
require_real_directory,
)
from openultrasast.mapping.entry_fact_matcher import EntryFactMatchResult, match_entry_fact
from openultrasast.mapping.entry_packs import load_installed_entry_fact_authority
from openultrasast.mapping.fact_authority import EntryFactAuthorityContext
from openultrasast.mapping.fact_match_store import FactMatchArtifactWriter
from openultrasast.mapping.installed_fact_authority import fact_pack_bytes
ENTRY_FACT_CONFORMANCE_SCHEMA = "openultrasast.access-control.entry-fact-conformance.v1"
_CORE_ENTRY_POINTS = (
"openultrasast.core.factapply.apply_fact",
"openultrasast.core.factmatch.validate_fact_match",
"openultrasast.core.factpack.validate_fact_pack",
"openultrasast.core.factvalidate.validate_fact_relations",
)
_SOURCE_CASES = {
"complete": "sources/complete.js",
"name_only": "sources/name-only.js",
"partial": "sources/partial.js",
"authority_substitution": "sources/complete.js",
"producer_reviewer_conflict": "sources/complete.js",
"resolver_byte_substitution": "sources/complete.js",
"lineage_omission": "sources/complete.js",
"custody_contamination": "sources/complete.js",
"legacy_projection_only": "sources/legacy-only.js",
"deterministic_replay": "sources/complete.js",
}
class EntryFactConformanceError(ValueError):
"""The target-owned conformance fixture cannot be reproduced safely."""
class _UnusedCustody:
def current_log(self, population_digest: str) -> provenance.PopulationCustodyLog:
raise AssertionError(f"authority load read custody: {population_digest}")
class _Custody:
def __init__(self, logs: tuple[provenance.PopulationCustodyLog, ...], policy: provenance.PopulationCustodyPolicy):
self.logs = {item.population_digest: item for item in logs}
self.policy = policy
def current_log(self, population_digest: str) -> provenance.PopulationCustodyLog:
try:
return self.logs[population_digest]
except KeyError as exc:
raise provenance.PopulationCustodyError("population custody log not found") from exc
def append(self, request: provenance.PopulationUseRequest) -> provenance.PopulationUseReceipt:
prior = self.current_log(request.population_digest)
custodian = self.policy.reviewed_custodian_digests[0]
receipt = provenance.record_population_use(request, prior, custodian, self.policy)
self.logs[request.population_digest] = provenance.append_population_use(prior, receipt, self.policy)
return receipt
@dataclass(frozen=True)
class _SubstitutingResolver:
delegate: canonical.ContentResolver
digest: str
def resolve(self, reference: canonical.ContentRef) -> canonical.ResolvedContent:
resolved = self.delegate.resolve(reference)
if reference.content_digest != self.digest:
return resolved
return canonical.ResolvedContent(b"!" + resolved.content[1:], resolved.receipt)
def _installed_entries() -> Path:
import openultrasast
return Path(openultrasast.__file__).resolve().parent / "ruleset" / "entries"
def _context() -> tuple[EntryFactAuthorityContext, str, _Custody]:
loaded = load_installed_entry_fact_authority(_UnusedCustody())
bundle = loaded.bundle
custodian = bundle.custody_policy.reviewed_custodian_digests[0]
logs: dict[str, provenance.PopulationCustodyLog] = {}
for lineage in bundle.lineages:
for use in lineage.uses:
prior = logs.setdefault(
use.source_population_digest,
provenance.empty_population_custody_log(use.source_population_digest, custodian),
)
request = provenance.PopulationUseRequest(
use.source_population_digest,
lineage.arm_id,
use.purpose,
use.labels_accessed,
use.influenced_artifact_digest,
prior.log_digest,
)
receipt = provenance.record_population_use(request, prior, custodian, bundle.custody_policy)
if receipt.receipt_digest != use.population_use_receipt_digest:
raise EntryFactConformanceError("installed lineage receipt cannot be reproduced")
logs[use.source_population_digest] = provenance.append_population_use(prior, receipt, bundle.custody_policy)
evaluation = canonical.content_digest("access-control-entry-conformance-population-v1")
logs[evaluation] = provenance.empty_population_custody_log(evaluation, custodian)
custody = _Custody(tuple(logs.values()), bundle.custody_policy)
return replace(loaded, custody=custody), evaluation, custody
def _read_sources(root: Path) -> dict[str, tuple[str, bytes]]:
values: dict[str, tuple[str, bytes]] = {}
try:
require_real_directory(root / "sources")
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError("sources require a real non-symlink directory") from exc
for case_id, relative in _SOURCE_CASES.items():
try:
content = read_regular_nofollow(root / relative)
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError(f"{case_id} requires a nonempty readable fixture") from exc
if not content:
raise EntryFactConformanceError(f"{case_id} requires a nonempty readable fixture")
values[case_id] = (relative, content)
return values
def _persist(output: Path, content: bytes, kind: str) -> dict[str, canonical.JsonValue]:
if not content:
raise EntryFactConformanceError("handoff artifacts must be nonempty")
digest = canonical.bytes_digest(content)
relative = f"artifacts/sha256/{digest}"
target = output / relative
try:
ensure_real_directory(output)
ensure_real_directory(output / "artifacts")
ensure_real_directory(target.parent)
persist_regular_nofollow(target, content)
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError("artifact path is unsafe or not a regular file") from exc
return {"kind": kind, "path": relative, "content_digest": digest, "byte_length": len(content)}
def _validator(name: str, status: str, reason: str | None = None) -> dict[str, canonical.JsonValue]:
return {"name": name, "status": status, "reason": reason}
def _case(
case_id: str,
source: tuple[str, bytes],
expected: str,
reason: str | None,
artifacts: tuple[dict[str, canonical.JsonValue], ...],
validators: tuple[dict[str, canonical.JsonValue], ...],
downstream_validators: tuple[dict[str, canonical.JsonValue], ...] = (),
) -> dict[str, canonical.JsonValue]:
path, content = source
return {
"case_id": case_id,
"fixture_path": path,
"fixture_digest": canonical.bytes_digest(content),
"fixture_byte_length": len(content),
"expected_status": expected,
"reason": reason,
"artifacts": artifacts,
"validators": validators,
"downstream_validators": downstream_validators,
}
def _match(root: Path, output: Path, context: EntryFactAuthorityContext, evaluation: str, relative: str) -> EntryFactMatchResult:
writer = FactMatchArtifactWriter(
output,
context.bundle.review_policy.trusted_store_identities[0],
canonical.content_digest("retain-access-control-conformance-v1"),
)
results = match_entry_fact(
repository_root=root,
relative_path=relative,
language="javascript",
fact_id="entry:express:app-route:get",
authority=context,
writer=writer,
evaluation_population_digest=evaluation,
)
if len(results) != 1:
raise EntryFactConformanceError("fixture must produce exactly one structural candidate")
return results[0]
def _match_artifacts(output: Path, result: EntryFactMatchResult) -> tuple[dict[str, canonical.JsonValue], ...]:
refs = tuple(item.evidence for item in result.match.observations) + (result.match_reference,)
return tuple(_persist(output, result.resolver.resolve(ref).content, "match-evidence") for ref in refs)
def _stamp(contract: str, evaluation: str, source_digest: str, result: EntryFactMatchResult) -> provenance.ProvenanceStamp:
return provenance.ProvenanceStamp(
canonical.content_digest("entry-conformance-analyzer"),
contract,
canonical.content_digest((result.pack.normalized_content_digest,)),
canonical.content_digest("entry-conformance-acquisition"),
canonical.content_digest("entry-conformance-configuration"),
evaluation,
source_digest,
)
def _record(result: EntryFactMatchResult, evaluation: str, source_digest: str) -> evidence.EvidenceRecord:
location = result.match.product_location
source = semantics.SemanticNode("constraint-source", "security_object", location)
target = semantics.SemanticNode("constraint-target", "operation", location)
product = semantics.EvidenceProvenance(semantics.ProvenanceKind.PRODUCT_CODE, source_digest, None, None, None)
constraint = semantics.make_relation(
kind=semantics.RelationKind.CONSTRAINT_EFFECT,
role="constraint",
source=source,
target=target,
location=location,
status=semantics.RelationStatus.INCOMPLETE,
conditions=(),
reasons=("missing_constraint_relation",),
provenance=product,
)
fields = (
semantics.FieldValue("operation", semantics.FieldKind.TEXT, "entry-registration"),
semantics.FieldValue("boundary", semantics.FieldKind.TEXT, "express"),
)
scope = evidence.AnalyzedScope("base", "head", source_digest, (evidence.SourceRegion(location.path, 1, 4),))
contract = result.pack.family_contract_digest
packet = evidence.EvidencePacket(
evidence.EVIDENCE_PACKET_SCHEMA,
result.pack.family_id,
contract,
family.derive_security_object_id(_context_contract().security_object, fields),
result.pack.ecosystem,
scope,
(*result.relations, constraint),
(),
fields,
("missing_constraint_relation",),
_stamp(contract, evaluation, source_digest, result),
)
decision = evidence.Decision(
"indeterminate",
evidence.NeutralOutcome.INDETERMINATE,
evidence.EvidenceBasis.DETERMINISTIC,
scope,
("missing_constraint_relation",),
evidence.derive_packet_digest(packet),
provenance.DecisionProvenance(None, None, None, None, None, None),
)
return evidence.EvidenceRecord(packet, decision, "")
def _context_contract() -> family.FamilyContract:
from openultrasast.semantic.obligations.core_contract import production_access_control_contract
return production_access_control_contract()
def _complete_case(
fixture_root: Path,
output: Path,
source: tuple[str, bytes],
context: EntryFactAuthorityContext,
evaluation: str,
) -> tuple[dict[str, canonical.JsonValue], EntryFactMatchResult]:
result = _match(fixture_root, output, context, evaluation, source[0])
record = _validate_complete(result, context, evaluation, source)
artifacts = _match_artifacts(output, result)
artifacts += (_persist(output, fact_pack_bytes(result.pack), "fact-pack"),)
artifacts += (_persist(output, evidence.evidence_record_bytes(record), "evidence-record"),)
validators = tuple(_validator(name, "passed") for name in _CORE_ENTRY_POINTS)
return _case("complete", source, "passed", None, artifacts, validators), result
def _validate_complete(
result: EntryFactMatchResult,
context: EntryFactAuthorityContext,
evaluation: str,
source: tuple[str, bytes],
) -> evidence.EvidenceRecord:
bundle = context.bundle
validate_fact_pack(
result.pack,
bundle.contract,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
result.resolver,
)
validate_fact_match(
result.pack,
result.match.fact_id,
result.match,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
result.resolver,
)
apply_fact(
result.pack,
result.match.fact_id,
result.match,
result.match_reference,
bundle.contract,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
result.resolver,
)
record = _record(result, evaluation, canonical.bytes_digest(source[1]))
_validate_record(record, result, context, evaluation)
return record
def _validate_record(
record: evidence.EvidenceRecord,
result: EntryFactMatchResult,
context: EntryFactAuthorityContext,
evaluation: str,
) -> None:
bundle = context.bundle
validate_fact_relations(
record,
bundle.contract,
bundle.packs,
evaluation,
bundle.lineages,
context.custody,
bundle.custody_policy,
bundle.review_policy,
result.resolver,
)
def build_entry_fact_conformance(fixture_root: Path, output_root: Path) -> bytes:
"""Run the target adapter and return its canonical immutable handoff manifest."""
fixture_root, output_root = Path(fixture_root), Path(output_root)
sources = _read_sources(fixture_root)
context, evaluation, _ = _context()
complete, result = _complete_case(fixture_root, output_root, sources["complete"], context, evaluation)
from openultrasast.mapping._entry_fact_conformance_negative import negative_cases
from openultrasast.mapping._entry_fact_conformance_replay import attach_replay, replay_material
cases = (complete, *negative_cases(fixture_root, output_root, sources, context, evaluation, result))
replay, dependencies = replay_material(output_root, context, evaluation, result, sources["complete"])
cases = attach_replay(tuple(sorted(cases, key=lambda item: str(item["case_id"]))), replay, dependencies)
payload: dict[str, canonical.JsonValue] = {
"schema_version": ENTRY_FACT_CONFORMANCE_SCHEMA,
"core_entry_points": _CORE_ENTRY_POINTS,
"cases": cases,
}
manifest = {**payload, "handoff_digest": canonical.content_digest(payload)}
return canonical.canonical_record_bytes(manifest, known_schema_versions=(ENTRY_FACT_CONFORMANCE_SCHEMA,))
def write_entry_fact_conformance(fixture_root: Path, output_root: Path) -> Path:
"""Generate artifacts and atomically-equivalent canonical manifest bytes."""
output = Path(output_root)
content = build_entry_fact_conformance(fixture_root, output)
target = output / "manifest.json"
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(content)
return target
__all__ = [
"ENTRY_FACT_CONFORMANCE_SCHEMA",
"EntryFactConformanceError",
"build_entry_fact_conformance",
"write_entry_fact_conformance",
]
@@ -0,0 +1,284 @@
"""Closed manifest validation for the downstream entry-fact handoff."""
from __future__ import annotations
import json
import re
from collections.abc import Mapping
from pathlib import Path, PurePosixPath
from typing import cast
from openultrasast.core import canonical
from openultrasast.mapping._entry_fact_conformance_fs import (
ConformanceFilesystemError,
flat_regular_census,
read_regular_nofollow,
require_real_directory,
)
from openultrasast.mapping.entry_fact_conformance import ENTRY_FACT_CONFORMANCE_SCHEMA, EntryFactConformanceError
_DIGEST = re.compile(r"[0-9a-f]{64}")
_TOP_FIELDS = {"schema_version", "handoff_digest", "core_entry_points", "cases"}
_CASE_FIELDS = {
"case_id",
"fixture_path",
"fixture_digest",
"fixture_byte_length",
"expected_status",
"reason",
"artifacts",
"validators",
"downstream_validators",
}
_ARTIFACT_FIELDS = {"kind", "path", "content_digest", "byte_length"}
_VALIDATOR_FIELDS = {"name", "status", "reason"}
_EXPECTED_CASES = {
"complete",
"name_only",
"partial",
"authority_substitution",
"producer_reviewer_conflict",
"resolver_byte_substitution",
"lineage_omission",
"custody_contamination",
"legacy_projection_only",
"deterministic_replay",
}
_CORE_ENTRY_POINTS = (
"openultrasast.core.factapply.apply_fact",
"openultrasast.core.factmatch.validate_fact_match",
"openultrasast.core.factpack.validate_fact_pack",
"openultrasast.core.factvalidate.validate_fact_relations",
)
_MATCH = "openultrasast.core.factmatch.validate_fact_match"
_MATRIX = {
"authority_substitution": ((_MATCH, "refused", "match_pack_identity_substituted"),),
"complete": tuple((name, "passed", None) for name in _CORE_ENTRY_POINTS),
"custody_contamination": ((_MATCH, "refused", "protected_population_contamination"),),
"deterministic_replay": ((_MATCH, "passed", None),),
"legacy_projection_only": ((_MATCH, "refused", "name_only_match"),),
"lineage_omission": ((_MATCH, "refused", "matcher_lineage_omitted"),),
"name_only": ((_MATCH, "refused", "missing_observation:receiver"),),
"partial": ((_MATCH, "refused", "missing_observation:arguments"),),
"producer_reviewer_conflict": (("openultrasast.core.factpack.validate_fact_pack", "refused", "self_authored_review_evidence"),),
"resolver_byte_substitution": ((_MATCH, "refused", "resolver_byte_substitution"),),
}
_DOWNSTREAM_AUTHORITY = "openultrasast.mapping.fact_authority.authenticate_entry_fact_authority"
_DOWNSTREAM_MATRIX: dict[str, tuple[tuple[str, str, str], ...]] = {case_id: () for case_id in _EXPECTED_CASES}
_DOWNSTREAM_MATRIX["authority_substitution"] = ((_DOWNSTREAM_AUTHORITY, "refused", "authority_bundle_digest_substituted"),)
_DOWNSTREAM_MATRIX["producer_reviewer_conflict"] = ((_DOWNSTREAM_AUTHORITY, "refused", "producer_reviewer_conflict"),)
def _freeze(value: object) -> canonical.JsonValue:
if value is None or isinstance(value, (str, bool, int, float)):
return value
if isinstance(value, list):
return tuple(_freeze(item) for item in value)
if isinstance(value, Mapping) and all(isinstance(key, str) for key in value):
return {cast(str, key): _freeze(item) for key, item in value.items()}
raise EntryFactConformanceError("manifest contains a value outside canonical JSON")
def _object(value: object, fields: set[str], name: str) -> Mapping[str, object]:
if not isinstance(value, Mapping) or set(value) != fields or not all(isinstance(key, str) for key in value):
raise EntryFactConformanceError(f"{name} has unknown or missing fields")
return cast(Mapping[str, object], value)
def _array(value: object, name: str) -> list[object]:
if not isinstance(value, list) or not value:
raise EntryFactConformanceError(f"{name} must be a nonempty array")
return value
def _list(value: object, name: str) -> list[object]:
if not isinstance(value, list):
raise EntryFactConformanceError(f"{name} must be an array")
return value
def _text(value: object, name: str) -> str:
if not isinstance(value, str) or not value:
raise EntryFactConformanceError(f"{name} must be nonempty text")
return value
def _digest(value: object, name: str) -> str:
text = _text(value, name)
if _DIGEST.fullmatch(text) is None:
raise EntryFactConformanceError(f"{name} must be a lowercase SHA-256 digest")
return text
def _positive(value: object, name: str) -> None:
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise EntryFactConformanceError(f"{name} must be a positive integer")
def _relative(value: object, name: str) -> None:
text = _text(value, name)
path = PurePosixPath(text)
if path.is_absolute() or ".." in path.parts or path.as_posix() != text:
raise EntryFactConformanceError(f"{name} must be a normalized relative path")
def _validate_case(value: object) -> str:
row = _object(value, _CASE_FIELDS, "case")
case_id = _text(row["case_id"], "case_id")
if case_id not in _MATRIX:
raise EntryFactConformanceError("case identity is unknown")
_relative(row["fixture_path"], "fixture_path")
_digest(row["fixture_digest"], "fixture_digest")
_positive(row["fixture_byte_length"], "fixture_byte_length")
status = _text(row["expected_status"], "expected_status")
reason = row["reason"]
if status not in {"passed", "refused"} or (reason is None) is not (status == "passed"):
raise EntryFactConformanceError("case status and stable reason are inconsistent")
if reason is not None:
_text(reason, "reason")
_validate_artifacts(row["artifacts"])
_validate_validators(case_id, row["validators"], status, reason)
_validate_downstream(case_id, row["downstream_validators"])
return case_id
def _validate_artifacts(value: object) -> None:
for artifact in _array(value, "artifacts"):
item = _object(artifact, _ARTIFACT_FIELDS, "artifact")
_text(item["kind"], "artifact.kind")
_relative(item["path"], "artifact.path")
digest = _digest(item["content_digest"], "artifact.content_digest")
if item["path"] != f"artifacts/sha256/{digest}":
raise EntryFactConformanceError("artifact path does not equal its digest address")
_positive(item["byte_length"], "artifact.byte_length")
def _validate_validators(case_id: str, value: object, status: str, reason: object) -> None:
actual_validators = []
for validator in _array(value, "validators"):
item = _object(validator, _VALIDATOR_FIELDS, "validator")
name = _text(item["name"], "validator.name")
if name not in _CORE_ENTRY_POINTS:
raise EntryFactConformanceError("validator is outside the exact core entry-point set")
validator_status = _text(item["status"], "validator.status")
if validator_status not in {"passed", "refused"}:
raise EntryFactConformanceError("validator status is unknown")
if item["reason"] is not None:
_text(item["reason"], "validator.reason")
if validator_status != status or item["reason"] != reason:
raise EntryFactConformanceError("validator result differs from its enclosing case")
actual_validators.append((name, validator_status, item["reason"]))
if tuple(actual_validators) != _MATRIX[case_id]:
raise EntryFactConformanceError("case validator matrix is incomplete or reordered")
def _validate_downstream(case_id: str, value: object) -> None:
actual = []
for validator in _list(value, "downstream_validators"):
item = _object(validator, _VALIDATOR_FIELDS, "downstream validator")
actual.append(
(
_text(item["name"], "downstream validator.name"),
_text(item["status"], "downstream validator.status"),
item["reason"],
)
)
if tuple(actual) != _DOWNSTREAM_MATRIX[case_id]:
raise EntryFactConformanceError("downstream validator matrix is incomplete or fabricated")
def _validate_files(root: Path, cases: list[object]) -> None:
declared_sources = {_text(_object(raw, _CASE_FIELDS, "case")["fixture_path"], "fixture_path") for raw in cases}
source_root = root / "sources"
try:
actual_sources = {f"sources/{name}" for name in flat_regular_census(source_root)}
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError("source filesystem census differs from declared regular fixtures") from exc
if actual_sources != declared_sources:
raise EntryFactConformanceError("source filesystem census differs from declared regular fixtures")
declared = _declared_artifact_paths(cases)
artifact_root = root / "artifacts" / "sha256"
try:
require_real_directory(root / "artifacts")
actual = {f"artifacts/sha256/{name}" for name in flat_regular_census(artifact_root)}
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError("artifact filesystem census differs from the manifest") from exc
if actual != declared:
raise EntryFactConformanceError("artifact filesystem census differs from the manifest")
_validate_materialized_bytes(root, cases)
def _declared_artifact_paths(cases: list[object]) -> set[str]:
declared: set[str] = set()
for raw in cases:
case = _object(raw, _CASE_FIELDS, "case")
for raw_artifact in _array(case["artifacts"], "artifacts"):
artifact = _object(raw_artifact, _ARTIFACT_FIELDS, "artifact")
declared.add(_text(artifact["path"], "artifact.path"))
return declared
def _safe_read(path: Path, kind: str) -> bytes:
try:
return read_regular_nofollow(path)
except ConformanceFilesystemError as exc:
raise EntryFactConformanceError(f"{kind} filesystem census differs from the manifest") from exc
def _validate_materialized_bytes(root: Path, cases: list[object]) -> None:
for raw in cases:
case = _object(raw, _CASE_FIELDS, "case")
fixture = root / _text(case["fixture_path"], "fixture_path")
fixture_content = _safe_read(fixture, "source")
if (
not fixture_content
or len(fixture_content) != case["fixture_byte_length"]
or canonical.bytes_digest(fixture_content) != case["fixture_digest"]
):
raise EntryFactConformanceError("fixture bytes do not match the manifest")
for raw_artifact in _array(case["artifacts"], "artifacts"):
artifact = _object(raw_artifact, _ARTIFACT_FIELDS, "artifact")
path = _text(artifact["path"], "artifact.path")
materialized = _safe_read(root / path, "artifact")
if (
not materialized
or len(materialized) != artifact["byte_length"]
or canonical.bytes_digest(materialized) != artifact["content_digest"]
):
raise EntryFactConformanceError("artifact bytes do not match the manifest")
def validate_entry_fact_conformance(content: bytes, root: Path | None = None) -> None:
"""Refuse noncanonical, open-schema, redirected, or self-inconsistent manifests."""
try:
parsed = json.loads(content)
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise EntryFactConformanceError("manifest is not canonical JSON") from exc
row = _object(parsed, _TOP_FIELDS, "manifest")
if row["schema_version"] != ENTRY_FACT_CONFORMANCE_SCHEMA:
raise EntryFactConformanceError("manifest schema is unknown")
frozen = cast(Mapping[str, canonical.JsonValue], _freeze(parsed))
if canonical.canonical_record_bytes(frozen, known_schema_versions=(ENTRY_FACT_CONFORMANCE_SCHEMA,)) != content:
raise EntryFactConformanceError("manifest bytes are not canonical")
entry_points = _array(row["core_entry_points"], "core_entry_points")
if not all(isinstance(item, str) and item for item in entry_points) or len(set(entry_points)) != len(entry_points):
raise EntryFactConformanceError("core entry points must be unique nonempty text")
core_entry_points = tuple(cast(str, item) for item in entry_points)
if core_entry_points != _CORE_ENTRY_POINTS:
raise EntryFactConformanceError("core entry points differ from the exact replay set")
cases = _array(row["cases"], "cases")
case_ids = tuple(_validate_case(item) for item in cases)
if set(case_ids) != _EXPECTED_CASES or case_ids != tuple(sorted(case_ids)):
raise EntryFactConformanceError("manifest cases are missing, duplicated, or unordered")
payload: Mapping[str, canonical.JsonValue] = {
"schema_version": ENTRY_FACT_CONFORMANCE_SCHEMA,
"core_entry_points": core_entry_points,
"cases": tuple(_freeze(item) for item in cases),
}
if _digest(row["handoff_digest"], "handoff_digest") != canonical.content_digest(payload):
raise EntryFactConformanceError("handoff digest cannot be re-derived")
if root is not None:
_validate_files(Path(root), cases)
__all__ = ["validate_entry_fact_conformance"]